/// <summary>
/// Converts the object to a NetMQMessage.
/// </summary>
/// <returns>Returns a new NetMQMessage instance.</returns>
public NetMQMessage ToNetMQMessage()
{
NetMQMessage message = new NetMQMessage();
message.Append(Topic);
if (Id != null)
{
message.Append(Id);
}
MemoryStream stream = new MemoryStream();
Data.WriteTo(stream);
NetMQFrame dataFrame = new NetMQFrame(stream.ToArray());
message.Append(dataFrame);
if (Bytes == null)
{
message.AppendEmptyFrame();
}
else
{
message.Append(Bytes);
}
return(message);
}
/// <summary>
/// process REQUEST from a client. MMI requests are implemented here directly
/// </summary>
/// <param name="sender">client identity</param>
/// <param name="message">the message received</param>
public void ProcessClientMessage ([NotNull] NetMQFrame sender, [NotNull] NetMQMessage message)
{
// should be
// REQUEST [service name][request] OR
// DISCOVER ['mmi.service'][service to discover]
if (message.FrameCount < 2)
throw new ArgumentException ("The message is malformed!");
var serviceFrame = message.Pop (); // [request] OR [service to discover]
var serviceName = serviceFrame.ConvertToString ();
var request = Wrap (sender, message); // [CLIENT ADR][e][request] OR [service name]
// if it is a "mmi.service" request, handle it locally
// this request searches for a service and returns a code indicating the result of that search
// OK => service exists and worker are available
// Pending => service exists but no workers are available
// Unknown => service does not exist
if (serviceName == "mmi.service")
{
var returnCode = MmiCode.Unknown;
var name = request.Last.ConvertToString ();
if (m_services.Exists (s => s.Name == name))
{
var svc = m_services.Find (s => s.Name == name);
returnCode = svc.DoWorkersExist () ? MmiCode.Ok : MmiCode.Pending;
}
// set the return code to be the last frame in the message
var rc = new NetMQFrame (returnCode.ToString ());// [return code]
request.RemoveFrame (message.Last); // [CLIENT ADR][e] -> [service name]
request.Append (serviceName); // [CLIENT ADR][e] <- ['mmi.service']
request.Append (rc); // [CLIENT ADR][e]['mmi.service'] <- [return code]
// remove client return envelope and insert
// protocol header and service name,
// then rewrap envelope
var client = UnWrap (request); // ['mmi.service'][return code]
request.Push (MDPConstants.MDP_CLIENT_HEADER); // [protocol header]['mmi.service'][return code]
var reply = Wrap (client, request); // [CLIENT ADR][e][protocol header]['mmi.service'][return code]
// send to back to CLIENT(!)
Socket.SendMultipartMessage (reply);
DebugLog ($"MMI request processed. Answered {reply}");
}
else
{
// get the requested service object
var service = ServiceRequired (serviceName);
// a standard REQUEST received
DebugLog ($"Dispatching -> {request} to {serviceName}");
// send to a worker offering the requested service
// will add command, header and worker adr envelope
ServiceDispatch (service, request); // [CLIENT ADR][e][request]
}
}
public NetMQMessage DecryptApplicationMessage([NotNull] NetMQMessage cipherMessage)
{
if (!SecureChannelReady)
{
throw new NetMQSecurityException(NetMQSecurityErrorCode.SecureChannelNotReady, "Cannot decrypt messages until the secure channel is ready");
}
if (cipherMessage == null)
{
throw new ArgumentNullException("cipherMessage");
}
if (cipherMessage.FrameCount < 2)
{
throw new NetMQSecurityException(NetMQSecurityErrorCode.InvalidFramesCount, "cipher message should have at least 2 frames");
}
NetMQFrame protocolVersionFrame = cipherMessage.Pop();
NetMQFrame contentTypeFrame = cipherMessage.Pop();
if (!protocolVersionFrame.ToByteArray().SequenceEqual(m_protocolVersion))
{
throw new NetMQSecurityException(NetMQSecurityErrorCode.InvalidProtocolVersion, "Wrong protocol version");
}
ContentType contentType = (ContentType)contentTypeFrame.Buffer[0];
if (contentType != ContentType.ApplicationData)
{
throw new NetMQSecurityException(NetMQSecurityErrorCode.InvalidContentType, "Not an applicagtion data message");
}
return(m_recordLayer.DecryptMessage(ContentType.ApplicationData, cipherMessage));
}
public override void SetFromNetMQMessage(NetMQMessage message)
{
base.SetFromNetMQMessage(message);
if (message.FrameCount != 3)
{
throw new NetMQSecurityException(NetMQSecurityErrorCode.InvalidFramesCount, "Malformed message");
}
// get the randon number
NetMQFrame randomNumberFrame = message.Pop();
RandomNumber = randomNumberFrame.ToByteArray();
// get the length of the ciphers array
NetMQFrame ciphersLengthFrame = message.Pop();
int ciphersLength = BitConverter.ToInt32(ciphersLengthFrame.Buffer, 0);
// get the ciphers
NetMQFrame ciphersFrame = message.Pop();
CipherSuites = new CipherSuite[ciphersLength];
for (int i = 0; i < ciphersLength; i++)
{
CipherSuites[i] = (CipherSuite)ciphersFrame.Buffer[i * 2 + 1];
}
}
protected void RaiseMessageReceived(NetMQFrame topic, NetMQFrame value)
{
var handler = MessageReceived;
MessageTopic mt = (MessageTopic)Enum.Parse(typeof(MessageTopic), topic.ConvertToString());
handler?.Invoke(this, new StockTVMessageReceivedEventArgs(mt, value.ToByteArray(true)));
}
/// <summary>
/// Remove the two frames from the given NetMQMessage, interpreting them thusly:
/// 1. a byte with the HandshakeType, assumed to be ClientKeyExchange
/// 2. a byte-array containing the EncryptedPreMasterSecret.
/// </summary>
/// <param name="message">a NetMQMessage - which must have 2 frames</param>
/// <exception cref="NetMQSecurityException"><see cref="NetMQSecurityErrorCode.InvalidFramesCount"/>: FrameCount must be 1.</exception>
public override void SetFromNetMQMessage(NetMQMessage message)
{
NetMQFrame lengthFrame = message.Pop();
NetMQFrame encryptedPreMasterSecretLengthFrame = message.Pop();
base.SetFromNetMQMessage(message);
}
public bool ProcessMessage(NetMQMessage incomingMessage, IList <NetMQMessage> outgoingMesssages)
{
ContentType contentType = ContentType.Handshake;
if (incomingMessage != null)
{
NetMQFrame protocolVersionFrame = incomingMessage.Pop();
byte[] protocolVersionBytes = protocolVersionFrame.ToByteArray();
if (protocolVersionBytes.Length != 2)
{
throw new NetMQSecurityException(NetMQSecurityErrorCode.InvalidFrameLength, "Wrong length for protocol version frame");
}
if (!protocolVersionBytes.SequenceEqual(m_protocolVersion))
{
throw new NetMQSecurityException(NetMQSecurityErrorCode.InvalidProtocolVersion, "Wrong protocol version");
}
NetMQFrame contentTypeFrame = incomingMessage.Pop();
if (contentTypeFrame.MessageSize != 1)
{
throw new NetMQSecurityException(NetMQSecurityErrorCode.InvalidFrameLength, "wrong length for message size");
}
contentType = (ContentType)contentTypeFrame.Buffer[0];
if (contentType != ContentType.ChangeCipherSpec && contentType != ContentType.Handshake)
{
throw new NetMQSecurityException(NetMQSecurityErrorCode.InvalidContentType, "Unkown content type");
}
if (ChangeSuiteChangeArrived)
{
incomingMessage = m_recordLayer.DecryptMessage(contentType, incomingMessage);
}
}
bool result = false;
if (contentType == ContentType.Handshake)
{
result = m_handshakeLayer.ProcessMessages(incomingMessage, m_outgoingMessageBag);
foreach (NetMQMessage outgoingMesssage in m_outgoingMessageBag.Messages)
{
outgoingMesssages.Add(outgoingMesssage);
}
m_outgoingMessageBag.Clear();
}
else
{
ChangeSuiteChangeArrived = true;
}
return(SecureChannelReady = result && ChangeSuiteChangeArrived);
}
private void RemoveLength(NetMQMessage incomingMessage)
{
if (Configuration.StandardTLSFormat)
{
//去除长度
NetMQFrame lengthFrame = incomingMessage.Pop();
}
}
/// <summary>
/// Remove the two frames from the given NetMQMessage, interpreting them thusly:
/// 1. a byte with the HandshakeType,
/// 2. a byte-array containing the X.509 digital certificate.
/// </summary>
/// <param name="message">a NetMQMessage - which must have 2 frames</param>
/// <exception cref="NetMQSecurityException"><see cref="NetMQSecurityErrorCode.InvalidFramesCount"/>: FrameCount must be 1.</exception>
public override void SetFromNetMQMessage(NetMQMessage message)
{
NetMQFrame lengthFrame = message.Pop();
NetMQFrame certslengthFrame = message.Pop();
NetMQFrame certlengthFrame = message.Pop();
base.SetFromNetMQMessage(message);
}
/// <summary>
/// check if the id and the id frame of the message are identical
/// </summary>
private static bool AreSame(byte[] id, NetMQFrame idFrame)
{
if (id.Length != idFrame.Buffer.Length)
{
return(false);
}
return(!id.Where((t, i) => t != idFrame.Buffer[i]).Any());
}
internal ZmqRequestMessage(NetMQFrame clientAddress, NetMQSocket responseChannel, TRequest requestData, ref IQueueLogger loggerObject)
{
#region Initialization
logger = loggerObject;
RequestData = requestData;
this.clientAddress = clientAddress;
this.responseChannel = responseChannel;
#endregion
}
/// <summary>
/// Wrap a message with the identity and an empty frame
/// </summary>
/// <returns>new created message</returns>
private static NetMQMessage Wrap(NetMQFrame identity, NetMQMessage msg)
{
var result = new NetMQMessage(msg);
result.PushEmptyFrame();
result.Push(identity);
return(result);
}
/// <summary>
/// Prepend the message with an empty frame as separator and a frame
/// </summary>
/// <returns>new message with wrapped content</returns>
private static NetMQMessage Wrap(NetMQMessage msg, NetMQFrame frame)
{
var result = new NetMQMessage(msg);
result.Push(NetMQFrame.Empty); // according to MDP an empty frame is the separator
result.Push(frame); // the return address
return(result);
}
// Way OUT: Sending information
public void Server_SendReply(RouterSocket server, NetMQFrame clientAddress, ChatMessage message)
{
var messageToClient = new NetMQMessage();
messageToClient.Append(clientAddress);
messageToClient.AppendEmptyFrame();
messageToClient.Append(message.ToJson(), Encoding.UTF8);
server.SendMultipartMessage(messageToClient);
}
public void SpecifyEncoding()
{
var frame = new NetMQFrame("Hello", Encoding.UTF32);
// size should be 4 times the string length because of using utf32
Assert.Equal(20, frame.MessageSize);
Assert.Equal("Hello", frame.ConvertToString(Encoding.UTF32));
}
public void Handle(NetMQFrame[] sender, NetMQMessage message)
{
Log.Debug("[CommitHandler] Got a commit to write...");
var requestId = message.Pop();
var commit = getCommit(message);
var task = _writer.Store(commit);
var commitContinuationContext = new CommitContinuationContext(sender, commit.CommitId, requestId);
task.ContinueWith(onComplete, commitContinuationContext, TaskContinuationOptions.ExecuteSynchronously);
Log.Debug("[CommitHandler] Commit queued up...");
}
/// <summary>
/// process REQUEST from a client. MMI requests are implemented here directly
/// </summary>
/// <param name="sender">client identity</param>
/// <param name="message">the message received</param>
public void ProcessClientMessage(NetMQFrame sender, NetMQMessage message)
{
// we expect [SERVICENAME][DATA]
if (message.FrameCount < 2)
{
throw new ArgumentException("The message is malformed!");
}
var serviceFrame = message.Pop(); // [service name][request] OR ["mmi.service"][service name]
var serviceName = serviceFrame.ConvertToString();
var service = ServiceRequired(serviceName);
var request = Wrap(sender, message); // [CLIENT ADR][e][request] OR [service name]
// if it is a "mmi.service" request, handle it locally
// this request searches for a service and returns a code indicating the result of that search
// 200 => service exists and worker are available
// 400 => service exists but no workers are available
// 501 => service does not exist
if (serviceName == "mmi.service")
{
var returnCode = "501";
var name = request.Last.ConvertToString();
if (m_services.Exists(s => s.Name == name))
{
var svc = m_services.Find(s => s.Name == name);
returnCode = svc.DoWorkersExist() ? "200" : "400"; // [CLIENT ADR][e][service name]
}
// set the return code to be the last frame in the message
var rc = new NetMQFrame(returnCode);
request.RemoveFrame(message.Last); // [CLIENT ADR][e]
request.Append(rc); // [CLIENT ADR][e][return code]
// remove client return envelope and insert
// protocol header and service name,
// then rewrap envelope
var client = UnWrap(message); // [return code]
request.Push(MDPClientHeader); // [protocol header][return code]
var reply = Wrap(client, request); // [CLIENT ADR][e][protocol header][return code]
// send to back to CLIENT(!)
Socket.SendMessage(reply);
Log(string.Format("[BROKER] MMI request processed. Answered {0}", reply));
}
else
{
// a standard REQUEST received
Log(string.Format("[BROKER] Dispatching request -> {0}", request));
// send to a worker offering the requested service
// will add command, header and worker adr evenlope
ServiceDispatch(service, request); // [CLIENT ADR][e][request]
}
}
/// <summary>
/// Remove the two frames from the given NetMQMessage, interpreting them thusly:
/// 1. a byte with the HandshakeType, assumed to be ClientKeyExchange
/// 2. a byte-array containing the EncryptedPreMasterSecret.
/// </summary>
/// <param name="message">a NetMQMessage - which must have 2 frames</param>
/// <exception cref="NetMQSecurityException"><see cref="NetMQSecurityErrorCode.InvalidFramesCount"/>: FrameCount must be 1.</exception>
public override void SetFromNetMQMessage(NetMQMessage message)
{
if (message.FrameCount != 1)
{
throw new NetMQSecurityException(NetMQSecurityErrorCode.InvalidFramesCount, "Malformed message");
}
NetMQFrame preMasterSecretFrame = message.Pop();
EncryptedPreMasterSecret = preMasterSecretFrame.ToByteArray();
}
/// <summary>
/// upon arrival of a message process it
/// and set the request variable accordingly
/// </summary>
/// <remarks>
/// worker expects to receive either of the following
/// REQUEST -> [e][header][command][client adr][e][request]
/// HEARTBEAT -> [e][header][command]
/// DISCONNECT -> [e][header][command]
/// KILL -> [e][header][command]
/// </remarks>
protected virtual void ProcessReceiveReady(object sender, NetMQSocketEventArgs e)
{
// a request has arrived process it
var request = m_worker.ReceiveMultipartMessage();
m_receivedMessage = true;
Log($"[WORKER] received {request}");
// any message from broker is treated as heartbeat(!) since communication exists
m_liveliness = _heartbeat_liveliness;
// check the expected message envelope and get the embedded MPD command
var command = GetMDPCommand(request);
// MDP command is one byte!
switch (command)
{
case MDPCommand.Request:
// the message is [client adr][e][request]
// save as many addresses as there are until we hit an empty frame
// - for simplicity assume it is just one
m_returnIdentity = Unwrap(request); // TODO give this responsibility to outside?!
if (request.FrameCount == 2) // TODO Improve this definition!! Maybe use another request type!? RequestCorrelated
{
m_requestId = request.Last;
request.RemoveFrame(m_requestId);
}
else
{
m_requestId = null;
}
OnWork(request);
break;
case MDPCommand.Heartbeat:
// reset the liveliness of the broker
m_liveliness = _heartbeat_liveliness;
break;
case MDPCommand.Disconnect:
// reconnect the worker
Connect();
break;
case MDPCommand.Kill:
// stop working you worker you
// m_exit = true; // TODO!
break;
default:
Log("[WORKER ERROR] invalid command received!");
break;
}
}
/// <summary>
/// adds an empty frame and a specified frame to a message
/// </summary>
private static NetMQMessage Wrap(NetMQFrame frame, NetMQMessage message)
{
var result = new NetMQMessage(message);
if (frame.BufferSize > 0)
{
result.Push(NetMQFrame.Empty);
result.Push(frame);
}
return(result);
}
// Way OUT: Sending information
internal void Server_BroadcastMessage(List <string> clientList, RouterSocket router,
string rawAddress, ChatMessage chatMessage)
{
foreach (var c in clientList)
{
if (rawAddress != c)
{
NetMQFrame anotherClientAddress = new NetMQFrame(c);
this.Server_SendReply(router, anotherClientAddress, chatMessage);
}
}
}
public void Handle(NetMQFrame[] sender, NetMQMessage message)
{
Logger.Debug("[Queue_SubscribeHandler] Received subscribe request.");
var requestId = message.Pop();
var context = message.Pop().ConvertToString();
var queueId = message.Pop().ConvertToString();
var subscriberId = message.Pop().ConvertToString();
var filter = message.Pop().ConvertToString();
var utcStartTime = message.PopDateTime();
var allocationSize = message.PopInt32();
var allocationTimeInMilliseconds = message.PopInt32();
var subscribe = new SubscribeToQueue(context,
queueId,
subscriberId,
filter,
utcStartTime, allocationSize, allocationTimeInMilliseconds);
var queuedEvents = _storage.Subscribe(subscribe);
var events = queuedEvents.Events;
var msg = new NetMQMessage();
msg.Append(sender);
msg.AppendEmptyFrame();
msg.Append(ResProtocol.ResClient01);
msg.Append(requestId);
msg.Append(ResCommands.QueuedEvents);
msg.Append(context);
msg.Append(queueId);
msg.Append(subscriberId);
msg.Append(DateTime.UtcNow.ToNetMqFrame());
msg.Append(queuedEvents.AllocationId.ToNetMqFrame());
var count = events.Length;
msg.Append(count.ToNetMqFrame());
foreach (var e in events)
{
msg.Append(e.EventId.ToByteArray());
msg.Append(e.Stream);
msg.Append(e.Context);
msg.Append(e.Sequence.ToNetMqFrame());
msg.Append(e.Timestamp.ToNetMqFrame());
msg.Append(e.TypeKey);
msg.Append(e.Headers.ToNetMqFrame());
msg.Append(e.Body);
}
var result = new QueuedMessagesFetched(msg);
while (!_outBuffer.Offer(result))
_spin.SpinOnce();
}
/// <summary>
/// Remove the two frames from the given NetMQMessage, interpreting them thusly:
/// 1. a byte with the HandshakeType,
/// 2. a byte-array containing the verification data - used to verify the integrity of the content.
/// </summary>
/// <param name="message">a NetMQMessage - which must have 2 frames</param>
/// <exception cref="NetMQSecurityException"><see cref="NetMQSecurityErrorCode.InvalidFramesCount"/>: FrameCount must be 1.</exception>
public override void SetFromNetMQMessage(NetMQMessage message)
{
base.SetFromNetMQMessage(message);
if (message.FrameCount != 1)
{
throw new NetMQSecurityException(NetMQSecurityErrorCode.InvalidFramesCount, "Malformed message");
}
NetMQFrame verifyDataFrame = message.Pop();
VerifyData = verifyDataFrame.ToByteArray();
}
private static void ServerThread()
{
using (var context = NetMQContext.Create())
using (var socket = context.CreateResponseSocket())
{
socket.Bind("tcp://*:8989");
NetMQFrame request = new NetMQFrame(socket.Receive());
Console.WriteLine(Encoding.UTF8.GetString(request.Buffer));
socket.Send(new NetMQFrame(Encoding.UTF8.GetBytes("World")).Buffer);
}
}
//called for each data set
public void sendData(T data)
{
msg = new NetMQMessage();
System.IO.MemoryStream stream = new System.IO.MemoryStream();
ProtoBuf.Serializer.Serialize <T>(stream, data);
NetMQFrame frame = new NetMQFrame(stream.ToArray());
//lastData = ProtoBuf.Serializer.Deserialize<T>(new System.IO.MemoryStream(frame.ToByteArray()));
msg.Push(frame);
publisher.SendMessage(msg);
}
private static void ServerThread()
{
using (var context = NetMQContext.Create())
using (var socket = context.CreateResponseSocket())
{
socket.Bind("tcp://*:8989");
NetMQFrame request = new NetMQFrame(socket.Receive());
Console.WriteLine(Encoding.UTF8.GetString(request.Buffer));
socket.Send(new NetMQFrame(Encoding.UTF8.GetBytes("World")).Buffer);
}
}
/// <summary>
/// return the the filename as the first item and the file data as the second item.
/// </summary>
/// <param name="fileInfo"></param>
/// <returns></returns>
private static Tuple <NetMQFrame, NetMQFrame> GetFrames(FileInfo fileInfo)
{
if (fileInfo.Exists)
{
var fileNameFrame = new NetMQFrame(fileInfo.Name);
var fileData = File.ReadAllBytes(fileInfo.FullName);
var fileDataFrame = new NetMQFrame(fileData);
return(new Tuple <NetMQFrame, NetMQFrame>(fileNameFrame, fileDataFrame));
}
else
{
throw new MethodFailedException();
}
}
/// <summary>
/// Remove the two frames from the given NetMQMessage, interpreting them thusly:
/// 1. a byte with the HandshakeType,
/// 2. a byte-array containing the X.509 digital certificate.
/// </summary>
/// <param name="message">a NetMQMessage - which must have 2 frames</param>
/// <exception cref="NetMQSecurityException"><see cref="NetMQSecurityErrorCode.InvalidFramesCount"/>: FrameCount must be 1.</exception>
public override void SetFromNetMQMessage(NetMQMessage message)
{
if (message.FrameCount != 1)
{
throw new NetMQSecurityException(NetMQSecurityErrorCode.InvalidFramesCount, "Malformed message");
}
NetMQFrame certificateFrame = message.Pop();
byte[] certificateBytes = certificateFrame.ToByteArray();
Certificate = new X509Certificate2();
Certificate.Import(certificateBytes);
}
public void GetResult_StringString_ShouldReturnExpectedResult()
{
const string expected_phrase = "Thank God Its Friday";
var replyFrame = new NetMQFrame(expected_phrase);
var fakeMDPClient = new MDPTestClientForTitanicClient {
ReplyDataFrame = replyFrame
};
var sut = new TitanicClient(fakeMDPClient);
var result = sut.GetResult("echo", expected_phrase);
result.Item2.Should().Be(TitanicReturnCode.Ok);
result.Item1.Should().BeSameAs(replyFrame.Buffer);
}
public void Reply_ExistingRequestStringIntegerInteger_ShouldReturnExpectedReply()
{
const string expected_phrase = "Thank God Its Friday";
var replyFrame = new NetMQFrame(expected_phrase);
var fakeMDPClient = new MDPTestClientForTitanicClient {
ReplyDataFrame = replyFrame
};
var sut = new TitanicClient(fakeMDPClient);
var reply = sut.Reply(Guid.NewGuid(), 0, TimeSpan.FromMilliseconds(0));
reply.Item2.Should().Be(TitanicReturnCode.Ok);
Encoding.UTF8.GetString(reply.Item1).Should().Be(expected_phrase);
}
private void SendErrorMessage(string messageText, ErrorCode errorCode, NetMQFrame idFrame)
{
Message <ErrorMessage> errorMessage = new Message <ErrorMessage>("Error")
{
Data = new ErrorMessage
{
MessageText = messageText,
ErrorCode = (int)errorCode
},
Id = idFrame
};
NetMQMessage msg = errorMessage.ToNetMQMessage();
SendMessage(msg);
}
/// <summary>
/// Transforms the message received on the Router socket to a message format
/// that is sent over the Dealer socket.
/// </summary>
/// <param name="message">Message to be sent.</param>
/// <returns>The transformed message.</returns>
public NetMQMessage ToDealerMessage(NetMQMessage message)
{
NetMQFrame idFrame = message[0];
NetMQFrame topicFrame = message[2];
NetMQFrame dataFrame = message[3];
NetMQFrame byteFrame = message[4];
NetMQMessage dealerMessage = new NetMQMessage();
dealerMessage.Append(topicFrame);
dealerMessage.Append(idFrame);
dealerMessage.Append(dataFrame);
dealerMessage.Append(byteFrame);
return(dealerMessage);
}
public void Send(TData request)
{
using (var socket = _context.CreatePushSocket())
{
socket.Bind(_bindEndPoint);
var envelope = new NetMQFrame(Encoding.UTF8.GetBytes(request.ToString()));
var body = new NetMQFrame(request.ToByteArray());
var msq = new NetMQMessage();
msq.Append(envelope);
msq.Append(body);
socket.SendMessage(msq);
}
}
public void Handle(NetMQFrame[] sender, NetMQMessage message)
{
Logger.Debug("[Query_LoadEventsByStream] Received a request.");
var requestId = message.Pop();
var context = message.Pop().ConvertToString();
var stream = message.Pop().ConvertToString();
var fromVersion = message.PopInt64();
var maxVersion = message.PopNullableInt64();
var events = _storage.LoadEventsForStream(context, stream, fromVersion, maxVersion);
var msg = new NetMQMessage();
msg.Append(sender);
msg.AppendEmptyFrame();
msg.Append(ResProtocol.ResClient01);
msg.Append(requestId);
msg.Append(ResCommands.QueryEventsByStreamResponse);
var count = events.Length;
msg.Append(count.ToNetMqFrame());
foreach (var e in events)
{
msg.Append(e.EventId.ToByteArray());
msg.Append(e.Stream);
msg.Append(e.Context);
msg.Append(e.Sequence.ToNetMqFrame());
msg.Append(e.Timestamp.ToNetMqFrame());
msg.Append(e.TypeKey);
msg.Append(e.Headers.ToNetMqFrame());
msg.Append(e.Body);
}
var result = new QueryEventsForStreamLoaded(msg);
while (!_buffer.Offer(result))
_spin.SpinOnce();
}
public bool Send(NetMQSocket mailbox)
{
var routingId = _routingId;
_routingId = null;
var message = Encode(this);
if (message != null)
{
if (mailbox is RouterSocket)
{
message.Push(routingId);
}
mailbox.TrySendMultipartMessage(TimeSpan.Zero, message);
return true;
}
return false;
}
public void SendAck(NetMQMessage reply)
{
if (ReferenceEquals(reply, null))
{
throw new ArgumentNullException("reply");
}
if (ReferenceEquals(m_returnIdentity, null))
{
throw new Exception("You can only send a message after receiving a reply!");
}
var message = Wrap(reply, m_returnIdentity);
if (m_requestId != null)
{
message.Append(m_requestId);
}
m_returnIdentity = null;
m_requestId = null;
m_pollerQueue.Enqueue(() => Send(MDPCommand.Reply, null, message));
}
private static string ReadOutFrame(NetMQFrame frame)
{
string test = null;
if (frame.BufferSize == 4)
test = BitConverter.ToInt32(frame.Buffer, 0).ToString();
else if (frame.BufferSize == 8)
test = BitConverter.ToInt64(frame.Buffer, 0).ToString();
// else if (msg[i].BufferSize > 800000)
else if (frame.BufferSize > 5000)
test = "PNG " + frame.BufferSize;
else
test = frame.ConvertToString(System.Text.Encoding.ASCII);
return test;
}
/// <summary>
/// adds an empty frame and a specified frame to a message
/// </summary>
private static NetMQMessage Wrap (NetMQFrame frame, NetMQMessage message)
{
var result = new NetMQMessage (message);
if (frame.BufferSize > 0)
{
result.Push (NetMQFrame.Empty);
result.Push (frame);
}
return result;
}
public static NetMQMessage Encode(ZreMessage msg)
{
var message = new NetMQMessage();
var frameSize = 2 + 1; // Signature and message ID
switch (msg._id)
{
case ZreMessageType.Hello:
// version is a 1-byte integer
frameSize += 1;
// sequence is a 2-byte integer
frameSize += 2;
// endpoint is a string with 1-byte length
frameSize++; // Size is one octet
if (!string.IsNullOrEmpty(msg._endpoint))
{
frameSize += msg._endpoint.Length;
}
// groups is an array of strings
frameSize += 4; // Size is 4 octets
if (msg._groups != null && msg._groups.Count > 0)
{
// Add up size of list contents
foreach (var @group in msg._groups)
{
frameSize += 4 + @group.Length;
}
}
// status is a 1-byte integer
frameSize += 1;
// name is a string with 1-byte length
frameSize++; // Size is one octet
if (!string.IsNullOrEmpty(msg._name))
{
frameSize += msg._name.Length;
}
// headers is an array of key=value strings
frameSize += 4; // Size is 4 octets
if (msg._headers != null && msg._headers.Count > 0)
{
msg._headersBytes = 0;
// Add up size of dictionary contents
foreach (var header in msg._headers)
{
msg._headersBytes += 1 + header.Key.Length;
msg._headersBytes += 4 + header.Value.Length;
}
}
frameSize += msg._headersBytes;
break;
case ZreMessageType.Whisper:
// version is a 1-byte integer
frameSize += 1;
// sequence is a 2-byte integer
frameSize += 2;
break;
case ZreMessageType.Shout:
// version is a 1-byte integer
frameSize += 1;
// sequence is a 2-byte integer
frameSize += 2;
// group is a string with 1-byte length
frameSize++; // Size is one octet
if (!string.IsNullOrEmpty(msg._group))
{
frameSize += msg._group.Length;
}
break;
case ZreMessageType.Join:
// version is a 1-byte integer
frameSize += 1;
// sequence is a 2-byte integer
frameSize += 2;
// group is a string with 1-byte length
frameSize++; // Size is one octet
if (!string.IsNullOrEmpty(msg._group))
{
frameSize += msg._group.Length;
}
// status is a 1-byte integer
frameSize += 1;
break;
case ZreMessageType.Leave:
// version is a 1-byte integer
frameSize += 1;
// sequence is a 2-byte integer
frameSize += 2;
// group is a string with 1-byte length
frameSize++; // Size is one octet
if (!string.IsNullOrEmpty(msg._group))
{
frameSize += msg._group.Length;
}
// status is a 1-byte integer
frameSize += 1;
break;
case ZreMessageType.Ping:
// version is a 1-byte integer
frameSize += 1;
// sequence is a 2-byte integer
frameSize += 2;
break;
case ZreMessageType.PingOk:
// version is a 1-byte integer
frameSize += 1;
// sequence is a 2-byte integer
frameSize += 2;
break;
}
var frame = new NetMQFrame(frameSize);
msg._needle = frame.ToByteArray();
msg._needleReader = new BinaryReader(new MemoryStream(msg._needle));
msg._needleWriter = new BinaryWriter(new MemoryStream(msg._needle));
msg._needleWriter.PutNumber2(0xAAA0 | ZreConstants.ProtocolSignature);
msg._needleWriter.PutNumber1((byte)msg._id);
switch (msg._id)
{
case ZreMessageType.Hello:
msg._needleWriter.PutNumber1(ZreConstants.ProtocolVersion);
msg._needleWriter.PutNumber2(msg._sequence);
if (!string.IsNullOrEmpty(msg._endpoint))
{
msg._needleWriter.PutString(msg._endpoint);
}
else
{
msg._needleWriter.PutNumber1(0); // Empty string
}
if (msg._groups != null)
{
msg._needleWriter.PutNumber4(msg._groups.Count);
foreach (var @group in msg._groups)
{
msg._needleWriter.PutLongString(@group);
}
}
else
{
msg._needleWriter.PutNumber4(0); // Empty string array
}
msg._needleWriter.PutNumber1(msg._status);
if (!string.IsNullOrEmpty(msg._name))
{
msg._needleWriter.PutString(msg._name);
}
else
{
msg._needleWriter.PutNumber1(0); // Empty string
}
if (msg._headers != null)
{
msg._needleWriter.PutNumber4(msg._headers.Count);
foreach (var header in msg._headers)
{
msg._needleWriter.PutString(header.Key);
msg._needleWriter.PutLongString(header.Value);
}
}
else
{
msg._needleWriter.PutNumber4(0); // Empty dictionary
}
break;
case ZreMessageType.Whisper:
msg._needleWriter.PutNumber1(ZreConstants.ProtocolVersion);
msg._needleWriter.PutNumber2(msg._sequence);
break;
case ZreMessageType.Shout:
msg._needleWriter.PutNumber1(ZreConstants.ProtocolVersion);
msg._needleWriter.PutNumber2(msg._sequence);
if (!string.IsNullOrEmpty(msg._group))
{
msg._needleWriter.PutString(msg._group);
}
else
{
msg._needleWriter.PutNumber1(0); // Empty string
}
break;
case ZreMessageType.Join:
msg._needleWriter.PutNumber1(ZreConstants.ProtocolVersion);
msg._needleWriter.PutNumber2(msg._sequence);
if (!string.IsNullOrEmpty(msg._group))
{
msg._needleWriter.PutString(msg._group);
}
else
{
msg._needleWriter.PutNumber1(0); // Empty string
}
msg._needleWriter.PutNumber1(msg._status);
break;
case ZreMessageType.Leave:
msg._needleWriter.PutNumber1(ZreConstants.ProtocolVersion);
msg._needleWriter.PutNumber2(msg._sequence);
if (!string.IsNullOrEmpty(msg._group))
{
msg._needleWriter.PutString(msg._group);
}
else
{
msg._needleWriter.PutNumber1(0); // Empty string
}
msg._needleWriter.PutNumber1(msg._status);
break;
case ZreMessageType.Ping:
msg._needleWriter.PutNumber1(ZreConstants.ProtocolVersion);
msg._needleWriter.PutNumber2(msg._sequence);
break;
case ZreMessageType.PingOk:
msg._needleWriter.PutNumber1(ZreConstants.ProtocolVersion);
msg._needleWriter.PutNumber2(msg._sequence);
break;
}
// Now send the data frame
message.Append(frame);
// Now send the message field if there is any
if (msg._id == ZreMessageType.Whisper)
{
if (msg._content != null && !msg._content.IsEmpty)
{
while (!msg._content.IsEmpty)
{
var contentFrame = msg._content.Pop();
message.Append(contentFrame);
}
}
else
{
message.AppendEmptyFrame();
}
}
// Now send the message field if there is any
if (msg._id == ZreMessageType.Shout)
{
if (msg._content != null && !msg._content.IsEmpty)
{
while (!msg._content.IsEmpty)
{
var contentFrame = msg._content.Pop();
message.Append(contentFrame);
}
}
else
{
message.AppendEmptyFrame();
}
}
return message;
}
/// <summary>
/// Prepend the message with an empty frame as separator and a frame
/// </summary>
/// <returns>new message with wrapped content</returns>
private NetMQMessage Wrap(NetMQMessage msg, NetMQFrame frame)
{
var result = new NetMQMessage(msg);
result.Push(NetMQFrame.Empty); // according to MDP an empty frame is the separator
result.Push(frame); // the return address
return result;
}
private void SendUdpFrame(NetMQFrame frame)
{
m_udpSocket.SendTo(frame.Buffer, 0, frame.MessageSize, SocketFlags.None, m_broadcastAddress);
}
/// <summary>
/// Wrap a message with the identity and an empty frame
/// </summary>
/// <returns>new created message</returns>
private static NetMQMessage Wrap(NetMQFrame identity, NetMQMessage msg)
{
var result = new NetMQMessage(msg);
result.PushEmptyFrame();
result.Push(identity);
return result;
}
/// <summary>
/// upon arrival of a message process it
/// and set the request variable accordingly
/// </summary>
/// <remarks>
/// worker expects to receive either of the following
/// REQUEST -> [e][header][command][client adr][e][request]
/// HEARTBEAT -> [e][header][command]
/// DISCONNECT -> [e][header][command]
/// KILL -> [e][header][command]
/// </remarks>
protected virtual void ProcessReceiveReady(object sender, NetMQSocketEventArgs e)
{
// a request has arrived process it
var request = m_worker.ReceiveMessage();
Log(string.Format("[WORKER] received {0}", request));
// make sure that we have no valid request yet!
// if something goes wrong we'll return 'null'
m_request = null;
// any message from broker is treated as heartbeat(!) since communication exists
m_liveliness = _HEARTBEAT_LIVELINESS;
// check the expected message envelope and get the embedded MPD command
var command = GetMDPCommand(request);
// MDP command is one byte!
switch (command)
{
case MDPCommand.Request:
// the message is [client adr][e][request]
// save as many addresses as there are until we hit an empty frame
// - for simplicity assume it is just one
m_returnIdentity = Unwrap(request);
// set the class variable in order to return the request to caller
m_request = request;
break;
case MDPCommand.Heartbeat:
// reset the liveliness of the broker
m_liveliness = _HEARTBEAT_LIVELINESS;
break;
case MDPCommand.Disconnect:
// reconnect the worker
Connect();
break;
case MDPCommand.Kill:
// stop working you worker you
m_exit = true;
break;
default:
Log("[WORKER ERROR] invalid command received!");
break;
}
}
public void GetResult_StringIntegerGeneric_ReturnExpectedResult ()
{
var expected = new TestEntity ();
var replyFrame = new NetMQFrame (expected.ConvertToBytes ());
var fakeMDPClient = new MDPTestClientForTitanicClient { ReplyDataFrame = replyFrame };
var sut = new TitanicClient (fakeMDPClient);
var result = sut.GetResult<TestEntity, TestEntity> ("echo", expected);
result.Item2.Should ().Be (TitanicReturnCode.Ok);
result.Item1.Id.Should ().Be (expected.Id);
result.Item1.Name.Should ().Be (expected.Name);
}
public void Reply_ExistingRequestStringTimeSpan_ShouldReturnExpectedReply ()
{
const string expected_phrase = "Thank God Its Friday";
var replyFrame = new NetMQFrame (expected_phrase);
var fakeMDPClient = new MDPTestClientForTitanicClient { ReplyDataFrame = replyFrame };
var sut = new TitanicClient (fakeMDPClient);
var reply = sut.Reply (Guid.NewGuid (), TimeSpan.FromMilliseconds (0));
reply.Item2.Should ().Be (TitanicReturnCode.Ok);
Encoding.UTF8.GetString (reply.Item1).Should ().Be (expected_phrase);
}
/// <summary>
/// process a READY, REPLY, HEARTBEAT, DISCONNECT message sent to the broker by a worker
/// </summary>
/// <param name="sender">the sender identity frame</param>
/// <param name="message">the message sent</param>
public void ProcessWorkerMessage (NetMQFrame sender, NetMQMessage message)
{
// should be
// READY [mdp command][service name]
// REPLY [mdp command][client adr][e][reply]
// HEARTBEAT [mdp command]
if (message.FrameCount < 1)
throw new ApplicationException ("Message with too few frames received.");
var mdpCommand = message.Pop (); // get the mdp command frame it should have only one byte payload
if (mdpCommand.BufferSize > 1)
throw new ApplicationException ("The MDPCommand frame had more than one byte!");
var cmd = (MDPCommand) mdpCommand.Buffer[0]; // [service name] or [client adr][e][reply]
var workerId = sender.ConvertToString (); // get the id of the worker sending the message
var workerIsKnown = m_knownWorkers.Any (w => w.Id == workerId);
switch (cmd)
{
case MDPCommand.Ready:
if (workerIsKnown)
{
// then it is not the first command in session -> WRONG
// if the worker is know to a service, remove it from that
// service and a potential waiting list therein
RemoveWorker (m_knownWorkers.Find (w => w.Id == workerId));
Log (string.Format ("[BROKER] READY out of sync. Removed worker {0}.", workerId));
}
else
{
// now a new - not know - worker sent his READY initiation message
// attach worker to service and mark as idle - worker has send READY as first message
var serviceName = message.Pop ().ConvertToString ();
var service = ServiceRequired (serviceName);
// create a new worker and set the service it belongs to
var worker = new Worker (workerId, sender, service);
// now add the worker
AddWorker (worker, service);
Log (string.Format ("[BROKER] READY processed. Worker {0} added to service {1}",
workerId,
serviceName));
}
break;
case MDPCommand.Reply:
if (workerIsKnown)
{
var worker = m_knownWorkers.Find (w => w.Id == workerId);
// remove the client return envelope and insert the protocol header
// and service name then rewrap the envelope
var client = UnWrap (message); // [reply]
message.Push (worker.Service.Name); // [service name][reply]
message.Push (MDPClientHeader); // [protocol header][service name][reply]
var reply = Wrap (client, message); // [client adr][e][protocol header][service name][reply]
Socket.SendMessage (reply);
Log (string.Format ("[BROKER] REPLY from {0} received and send to {1} -> {2}",
workerId,
client.ConvertToString (),
message));
// relist worker for further requests
AddWorker (worker, worker.Service);
}
break;
case MDPCommand.Heartbeat:
if (workerIsKnown)
{
var worker = m_knownWorkers.Find (w => w.Id == workerId);
worker.Expiry = DateTime.UtcNow + m_heartbeatExpiry;
Log (string.Format ("[BROKER] HEARTBEAT from {0} received.", workerId));
}
break;
default:
Log ("[BROKER] ERROR: Invalid MDPCommand received or message received!");
break;
}
}
private void OnPipeReady(object sender, NetMQSocketEventArgs e)
{
NetMQMessage message = m_pipe.ReceiveMultipartMessage();
string command = message.Pop().ConvertToString();
switch (command)
{
case ConfigureCommand:
string interfaceName = message.Pop().ConvertToString();
int port = message.Pop().ConvertToInt32();
Configure(interfaceName, port);
break;
case PublishCommand:
m_transmit = message.Pop();
m_pingTimer.Interval = message.Pop().ConvertToInt32();
m_pingTimer.Enable = true;
SendUdpFrame(m_transmit);
break;
case SilenceCommand:
m_transmit = null;
m_pingTimer.Enable = false;
break;
case SubscribeCommand:
m_filter = message.Pop();
break;
case UnsubscribeCommand:
m_filter = null;
break;
case NetMQActor.EndShimMessage:
m_poller.Cancel();
break;
default:
throw new ArgumentOutOfRangeException();
}
}
public void GetResult_StringStringEncoding_ShouldReturnExpectedResult ()
{
var enc = Encoding.Unicode;
const string expected_phrase = "Thank God Its Friday";
var replyFrame = new NetMQFrame (enc.GetBytes (expected_phrase));
var fakeMDPClient = new MDPTestClientForTitanicClient { ReplyDataFrame = replyFrame };
var sut = new TitanicClient (fakeMDPClient);
var result = sut.GetResult ("echo", expected_phrase, enc);
result.Item2.Should ().Be (TitanicReturnCode.Ok);
result.Item1.Should ().Be (enc.GetString (replyFrame.Buffer));
}
private NetMQFrame ReceiveUdpFrame(out string peerName)
{
var buffer = new byte[UdpFrameMax];
EndPoint peer = new IPEndPoint(IPAddress.Any, 0);
int bytesRead = m_udpSocket.ReceiveFrom(buffer, ref peer);
var frame = new NetMQFrame(bytesRead);
Buffer.BlockCopy(buffer, 0, frame.Buffer, 0, bytesRead);
peerName = peer.ToString();
return frame;
}
public void Reply_ExistingRequestStringIntegerTimeSpanGeneric_ShouldReturnExpectedReply ()
{
var expected = new TestEntity ();
var replyFrame = new NetMQFrame (expected.ConvertToBytes ());
var fakeMDPClient = new MDPTestClientForTitanicClient { ReplyDataFrame = replyFrame };
var sut = new TitanicClient (fakeMDPClient);
var reply = sut.Reply (Guid.NewGuid (), 0, TimeSpan.FromMilliseconds (0));
reply.Item2.Should ().Be (TitanicReturnCode.Ok);
var result = new TestEntity ();
result.GenerateFrom (reply.Item1);
result.Id.Should ().Be (expected.Id);
result.Name.Should ().Be (expected.Name);
}
/// <summary>
/// process REQUEST from a client. MMI requests are implemented here directly
/// </summary>
/// <param name="sender">client identity</param>
/// <param name="message">the message received</param>
public void ProcessClientMessage ([NotNull] NetMQFrame sender, [NotNull] NetMQMessage message)
{
// should be
// REQUEST [service name][request] OR
// DISCOVER ['mmi.service'][service to discover]
if (message.FrameCount < 2)
throw new ArgumentException ("The message is malformed!");
var serviceFrame = message.Pop (); // [request] OR [service to discover]
var serviceName = serviceFrame.ConvertToString ();
var request = Wrap (sender, message); // [CLIENT ADR][e][request] OR [service name]
// if it is a "mmi.service" request, handle it locally
// this request searches for a service and returns a code indicating the result of that search
// OK => service exists and worker are available
// Pending => service exists but no workers are available
// Unknown => service does not exist
if (serviceName == "mmi.service")
{
var returnCode = MmiCode.Unknown;
var name = request.Last.ConvertToString ();
if (m_services.Exists (s => s.Name == name))
{
var svc = m_services.Find (s => s.Name == name);
returnCode = svc.DoWorkersExist () ? MmiCode.Ok : MmiCode.Pending;
}
// set the return code to be the last frame in the message
var rc = new NetMQFrame (returnCode.ToString ());// [return code]
request.RemoveFrame (message.Last); // [CLIENT ADR][e] -> [service name]
request.Append (serviceName); // [CLIENT ADR][e] <- ['mmi.service']
request.Append (rc); // [CLIENT ADR][e]['mmi.service'] <- [return code]
// remove client return envelope and insert
// protocol header and service name,
// then rewrap envelope
var client = UnWrap (request); // ['mmi.service'][return code]
request.Push (MDPClientHeader); // [protocol header]['mmi.service'][return code]
var reply = Wrap (client, request); // [CLIENT ADR][e][protocol header]['mmi.service'][return code]
// send to back to CLIENT(!)
Socket.SendMessage (reply);
DebugLog (string.Format ("MMI request processed. Answered {0}", reply));
}
else
{
// get the requested service object
var service = ServiceRequired (serviceName);
// a standard REQUEST received
DebugLog (string.Format ("Dispatching -> {0} to {1}", request, serviceName));
// send to a worker offering the requested service
// will add command, header and worker adr envelope
ServiceDispatch (service, request); // [CLIENT ADR][e][request]
}
}
/// <summary>
/// check if the id and the id frame of the message are identical
/// </summary>
private static bool AreSame(byte[] id, NetMQFrame idFrame)
{
if (id.Length != idFrame.Buffer.Length)
return false;
return !id.Where((t, i) => t != idFrame.Buffer[i]).Any();
}
private static MethodInfo UnPackNetMQFrame(
int FrameCount,
ref bool stopSignal,
BinarySerializer serializer,
NetMQFrame frame,
MethodInfo methodinfo,
List<object> methodParameters,
ref bool typeParameter,
ref Type type)
{
string address;
byte[] messageAsBytes;
int numberOfParameters;
if (FrameCount == 0)
{
address = serializer.GetString(frame.Buffer);
}
if (FrameCount == 1)
{
messageAsBytes = frame.Buffer;
string stopMessage = serializer.GetString(messageAsBytes);
if (stopMessage.ToLower() == "stop")
{
stopSignal = true;
}
}
if (FrameCount == 2)
{
methodinfo = (MethodInfo)serializer.Deserializer(frame.Buffer, typeof(MethodInfo));
}
if (FrameCount == 3)
{
numberOfParameters = int.Parse(serializer.GetString(frame.Buffer).Replace("ParameterCount:", string.Empty));
}
if (FrameCount > 3)
{
if (typeParameter)
{
type = (Type)serializer.Deserializer(frame.Buffer, typeof(Type));
typeParameter = false;
}
else
{
var parameter = serializer.Deserializer(frame.Buffer, type);
methodParameters.Add(parameter);
typeParameter = true;
}
}
return methodinfo;
}
/// <summary>
/// the broker setting up the cluster
///
///
/// State 2 ---+ +--- State n
/// | |
/// +----+----+
/// client 1 ---| | |--- worker 1
/// client 2 ---+---- BROKER 1 ----+--- worker 2
/// : | | | :
/// client n ---+ +----+----+ +--- worker n
/// | |
/// BROKER 2 BROKER n
///
/// BROKER 2 and n are not included and must be setup separately
///
/// A minimum of two address must be supplied
/// </summary>
/// <param name="args">[0] = this broker's address
/// [1] = 1st peer's address
/// :
/// [n] = nth peer address</param>
/// <remarks>
/// since "inproc://" is not working in NetMQ we use "tcp://"
/// for each broker we need 5 ports which for this example are
/// assigned as follows (in true life it should be configurable whether
/// they are ports or tcp/ip addresses)
///
/// this brokers address => local frontend binds to tcp://127.0.0.1:5555
/// cloud frontend binds to :5556
/// local backend binds to :5557
/// state backend binds to :5558
/// monitor PULL binds to :5559
///
/// the sockets are connected as follows
///
/// this broker's monitor PUSH connects to tcp://127.0.0.1:5559
///
/// (if peer's address and port is tcp://127.0.0.1:5575)
///
/// this broker's cloud backend connects to :5576
/// this broker's state frontend connects to :5578
///
/// this scheme is fix in this example
/// </remarks>
public static void Main(string[] args)
{
Console.Title = "NetMQ Inter-Broker Router";
const string baseAddress = "tcp://127.0.0.1:";
if (args.Length < 2)
{
Console.WriteLine("usage: program me peer1 [peer]*");
Console.WriteLine("each broker needs 5 port for his sockets!");
Console.WriteLine("place enough distance between multiple broker addresses!");
Environment.Exit(-1);
}
// trapping Ctrl+C as exit signal!
Console.CancelKeyPress += (s, e) =>
{
e.Cancel = true;
s_keepRunning = false;
};
// get random generator for later use
var rnd = new Random();
// get list for registering the clients
var clients = new List<byte[]>(NbrClients);
// get a list of peer addresses
var peers = new List<byte[]>();
// get all peer addresses - first is this broker!
for (var i = 1; i < args.Length; i++)
peers.Add(Encoding.UTF8.GetBytes(args[i]));
// build this broker's address
var me = baseAddress + args[0];
// get the port as integer for later use
var myPort = int.Parse(args[0]);
Console.WriteLine("[BROKER] The broker can be stopped by using CTRL+C!");
Console.WriteLine("[BROKER] setting up sockets ...");
// set up all the addresses needed in the due course
var localFrontendAddress = me;
var cloudFrontendAddress = baseAddress + (myPort + 1);
var localBackendAddress = baseAddress + (myPort + 2);
var stateBackendAddress = baseAddress + (myPort + 3);
var monitorAddress = baseAddress + (myPort + 4);
// create the context and all the sockets
using (var context = NetMQContext.Create())
using (var localFrontend = context.CreateRouterSocket())
using (var localBackend = context.CreateRouterSocket())
using (var cloudFrontend = context.CreateRouterSocket())
using (var cloudBackend = context.CreateRouterSocket())
using (var stateBackend = context.CreatePublisherSocket())
using (var stateFrontend = context.CreateSubscriberSocket())
using (var monitor = context.CreatePullSocket())
{
// give every socket an unique identity, e.g. LocalFrontend[Port]
SetIdentities(myPort,
localFrontend,
cloudFrontend,
localBackend,
stateBackend,
monitor,
cloudBackend, stateFrontend);
// subscribe to any message on the stateFrontend socket!
stateFrontend.Subscribe("");
// bind the serving sockets
localFrontend.Bind(localFrontendAddress);
cloudFrontend.Bind(cloudFrontendAddress);
localBackend.Bind(localBackendAddress);
stateBackend.Bind(stateBackendAddress);
monitor.Bind(monitorAddress);
// connect sockets to peers
for (var i = 1; i < args.Length; i++)
{
// build the cloud back end address
var peerPort = int.Parse(args[i]);
var address = baseAddress + (peerPort + 1);
Console.WriteLine("[BROKER] connect to cloud peer {0}", address);
// this cloudBackend connects to all peer cloudFrontends
cloudBackend.Connect(address);
// build the state front end address
address = baseAddress + (peerPort + 3);
Console.WriteLine("[BROKER] subscribe to state peer {0}", address);
// this stateFrontend to all peer stateBackends
stateFrontend.Connect(address);
}
// setup the local worker queue for LRU and monitor cloud capacity
var workerQueue = new Queue<byte[]>();
int previousLocalCapacity = 0;
// receive the capacity available from other peer(s)
stateFrontend.ReceiveReady += (s, e) =>
{
// the message should contain the available cloud capacity
var capacity = e.Socket.ReceiveFrameString();
Debug.Assert(string.IsNullOrWhiteSpace(capacity), "StateFrontend: message was empty!");
int couldCapacity;
Debug.Assert(int.TryParse(capacity, out couldCapacity), "StateFrontend: message did not contain a number!");
};
// get the status message and print it
monitor.ReceiveReady += (s, e) =>
{
var msg = e.Socket.ReceiveFrameString();
Console.WriteLine("[MONITOR] {0}", msg);
};
// all local clients are connecting to this socket
// they send a REQ and get a REPLY
localFrontend.ReceiveReady += (s, e) =>
{
// [client adr][empty][message id]
var request = e.Socket.ReceiveMultipartMessage();
// register the local client for later identification if not known
if (!clients.Any(n => AreSame(n, request[0])))
clients.Add(request[0].Buffer);
// if we have local capacity send worker else send to cloud
if (workerQueue.Count > 0)
{
// get the LRU worker adr
var worker = workerQueue.Dequeue();
// wrap message with workers address
var msg = Wrap(worker, request);
// send message to the worker
// [worker adr][empty][client adr][empty][data]
localBackend.SendMessage(msg);
}
else
{
// get an random index for peers
var peerIdx = rnd.Next(peers.Count - 2) + 2;
// get peers address
var peer = peers[peerIdx];
// wrap message with peer's address
var msg = Wrap(peer, request);
// [peer adr][empty][client adr][empty][data]
cloudBackend.SendMessage(msg);
}
};
// the workers are connected to this socket
// we get a REPLY either for a cloud client [worker adr][empty][peer adr][empty][peer client adr][empty][data]
// or local client [worker adr][empty][client adr][empty][data]
// or a READY message [worker adr][empty][WORKER_READY]
localBackend.ReceiveReady += (s, e) =>
{
// a worker can send "READY" or a request
// or an REPLAY
var msg = e.Socket.ReceiveMultipartMessage();
// just to make sure we received a proper message
Debug.Assert(msg != null && msg.FrameCount > 0, "[LocalBackend] message was empty or frame count == 0!");
// get the workers identity
var id = Unwrap(msg);
// this worker done in either way so add it to available workers
workerQueue.Enqueue(id);
// if it is NOT a ready message we need to route the message
// it could be a reply to a peer or a local client
// [WORKER_READY] or [client adr][empty][data] or [peer adr][empty][peer client adr][empty][data]
if (msg[0].Buffer[0] != WorkerReady)
{
Debug.Assert(msg.FrameCount > 2, "[LocalBackend] None READY message malformed");
// if the adr (first frame) is any of the clients send the REPLY there
// and send it to the peer otherwise
if (clients.Any(n => AreSame(n, msg.First)))
localFrontend.SendMessage(msg);
else
cloudFrontend.SendMessage(msg);
}
};
// this socket is connected to all peers
// we receive either a REQ or a REPLY form a peer
// REQ [peer adr][empty][peer client adr][empty][message id] -> send to peer for processing
// REP [peer adr][empty][client adr][empty][message id] -> send to local client
cloudBackend.ReceiveReady += (s, e) =>
{
var msg = e.Socket.ReceiveMultipartMessage();
// just to make sure we received a message
Debug.Assert(msg != null && msg.FrameCount > 0, "[CloudBackend] message was empty or frame count == 0!");
// we need the peers address for proper addressing
var peerAdr = Unwrap(msg);
// the remaining message must be at least 3 frames!
Debug.Assert(msg.FrameCount > 2, "[CloudBackend] message malformed");
// if the id is any of the local clients it is a REPLY
// and a REQ otherwise
if (clients.Any(n => AreSame(n, msg.First)))
{
// [client adr][empty][message id]
localFrontend.SendMessage(msg);
}
else
{
// add the peers address to the request
var request = Wrap(peerAdr, msg);
// [peer adr][empty][peer client adr][empty][message id]
cloudFrontend.SendMessage(request);
}
};
// all peers are binding to this socket
// we receive REPLY or REQ from peers
// REQ [peer adr][empty][peer client adr][empty][data] -> send to local worker for processing
// REP [peer adr][empty][client adr][empty][data] -> send to local client
cloudFrontend.ReceiveReady += (s, e) =>
{
var msg = e.Socket.ReceiveMultipartMessage();
// just to make sure we received a message
Debug.Assert(msg != null && msg.FrameCount > 0, "[CloudFrontend] message was empty or frame count == 0!");
// we may need need the peers address for proper addressing
var peerAdr = Unwrap(msg);
// the remaining message must be at least 3 frames!
Debug.Assert(msg.FrameCount > 2, "[CloudFrontend] message malformed");
// if the address is any of the local clients it is a REPLY
// and a REQ otherwise
if (clients.Any(n => AreSame(n, msg.First)))
localFrontend.SendMessage(msg);
else
{
// in order to know which per to send back the peers adr must be added again
var original = Wrap(peerAdr, msg);
// reduce the capacity to reflect the use of a worker by a cloud request
previousLocalCapacity = workerQueue.Count;
// get the LRU worker
var workerAdr = workerQueue.Dequeue();
// wrap the message with the worker address and send
var request = Wrap(workerAdr, original);
localBackend.SendMessage(request);
}
};
// in order to reduce chatter we only check to see if we have local capacity to provide to cloud
// periodically every 2 seconds with a timer
var timer = new NetMQTimer((int)TimeSpan.FromSeconds(2).TotalMilliseconds);
timer.Elapsed += (t, e) =>
{
// send message only if the previous send information changed
if (previousLocalCapacity != workerQueue.Count)
{
// set the information
previousLocalCapacity = workerQueue.Count;
// generate the message
var msg = new NetMQMessage();
var data = new NetMQFrame(previousLocalCapacity.ToString());
msg.Append(data);
var stateMessage = Wrap(Encoding.UTF8.GetBytes(me), msg);
// publish info
stateBackend.SendMessage(stateMessage);
}
// restart the timer
e.Timer.Enable = true;
};
// start all clients and workers as threads
var clientTasks = new Thread[NbrClients];
var workerTasks = new Thread[NbrWorker];
for (var i = 0; i < NbrClients; i++)
{
var client = new Client(localFrontendAddress, monitorAddress, (byte)i);
clientTasks[i] = new Thread(client.Run) { Name = string.Format("Client_{0}", i) };
clientTasks[i].Start();
}
for (var i = 0; i < NbrWorker; i++)
{
var worker = new Worker(localBackendAddress, (byte)i);
workerTasks[i] = new Thread(worker.Run) { Name = string.Format("Worker_{0}", i) };
workerTasks[i].Start();
}
var sockets = new NetMQSocket[]
{
localFrontend,
localBackend,
cloudFrontend,
cloudBackend,
stateFrontend,
stateBackend,
monitor
};
// create poller and add sockets & timer
var poller = new Poller(sockets);
poller.AddTimer(timer);
// start monitoring the sockets
Task.Factory.StartNew(poller.PollTillCancelled);
// we wait for a CTRL+C to exit
while (s_keepRunning)
{}
Console.WriteLine("Ctrl-C encountered! Exiting the program!");
if (poller.IsStarted)
poller.CancelAndJoin();
poller.Dispose();
}
}
public CommitContinuationContext(NetMQFrame[] sender, Guid commitId, NetMQFrame requestId)
{
Sender = sender;
CommitId = commitId;
RequestId = requestId;
}
public static void Append(this NetMQMessage message, NetMQFrame[] frames)
{
foreach (var frame in frames)
message.Append(frame);
}
public void SpecifyEncoding()
{
var frame = new NetMQFrame("Hello", Encoding.UTF32);
// size should be 4 times the string length because of using utf32
Assert.AreEqual(20, frame.MessageSize);
Assert.AreEqual("Hello", frame.ConvertToString(Encoding.UTF32));
}
public void Dispatch(string command, NetMQFrame[] sender, NetMQMessage message)
{
_handlers[command].Handle(sender, message);
}
public Worker(string id, NetMQFrame identity, Service service)
{
Id = id;
Identity = identity;
Service = service;
}
/// <summary>
/// upon arrival of a message process it
/// and set the request variable accordingly
/// </summary>
/// <remarks>
/// worker expects to receive either of the following
/// REQUEST -> [e][header][command][client adr][e][request]
/// HEARTBEAT -> [e][header][command]
/// DISCONNECT -> [e][header][command]
/// KILL -> [e][header][command]
/// </remarks>
protected virtual void ProcessReceiveReady(object sender, NetMQSocketEventArgs e)
{
// a request has arrived process it
var request = m_worker.ReceiveMultipartMessage();
m_receivedMessage = true;
Log($"[WORKER] received {request}");
// any message from broker is treated as heartbeat(!) since communication exists
m_liveliness = _heartbeat_liveliness;
// check the expected message envelope and get the embedded MPD command
var command = GetMDPCommand(request);
// MDP command is one byte!
switch (command)
{
case MDPCommand.Request:
// the message is [client adr][e][request]
// save as many addresses as there are until we hit an empty frame
// - for simplicity assume it is just one
m_returnIdentity = Unwrap(request); // TODO give this responsibility to outside?!
if (request.FrameCount == 2) // TODO Improve this definition!! Maybe use another request type!? RequestCorrelated
{
m_requestId = request.Last;
request.RemoveFrame(m_requestId);
}
else
{
m_requestId = null;
}
OnWork(request);
break;
case MDPCommand.Heartbeat:
// reset the liveliness of the broker
m_liveliness = _heartbeat_liveliness;
break;
case MDPCommand.Disconnect:
// reconnect the worker
Connect();
break;
case MDPCommand.Kill:
// stop working you worker you
// m_exit = true; // TODO!
break;
default:
Log("[WORKER ERROR] invalid command received!");
break;
}
}
