public void SaveMessage_ReplyMessage_SholdCreateNewFileWithGuidAsName ()
{
var sut = new TitanicFileIO (Path.GetTempPath ());
var message = new NetMQMessage ();
message.Push ("Hello World");
message.Push ("echo");
var messageSize = message[0].BufferSize + message[1].BufferSize + 4; // 2 lines with \r\n
var id = Guid.NewGuid ();
sut.SaveMessage (TitanicOperation.Reply, id, message);
var expectedDir = sut.TitanicDirectory;
var expectedFile = Path.Combine (expectedDir, id + _reply_ending);
File.Exists (expectedFile).Should ().BeTrue ("because the file exists");
var info = new FileInfo (expectedFile);
info.Length.Should ().Be (messageSize);
File.Delete (expectedFile);
}
public void ReceiveImplicitConnect_ValidScenario_ShouldReturnRequest()
{
const string hostAddress = "tcp://localhost:5557";
var loggingMessages = new List<string> ();
// setup the counter socket for communication
using (var context = NetMQContext.Create ())
using (var broker = context.CreateRouterSocket ())
using (var poller = new Poller ())
using (var session = new MDPWorker (hostAddress, "test", new[] { (byte) '1' }))
{
broker.Bind (hostAddress);
// we need to pick up any message in order to avoid errors
broker.ReceiveReady += (s, e) =>
{
var msg = e.Socket.ReceiveMultipartMessage ();
// we expect to receive a 5 Frame message
// [WORKER ADR][EMPTY]["MDPW01"]["READY"]["test"]
if (msg.FrameCount != 5)
Assert.Fail ("Message with wrong count of frames {0}", msg.FrameCount);
// make sure the frames are as expected
Assert.That (msg[1], Is.EqualTo (NetMQFrame.Empty));
Assert.That (msg[2].ConvertToString (), Is.EqualTo ("MDPW01"));
Assert.That (msg[3].BufferSize, Is.EqualTo (1));
Assert.That (msg[3].Buffer[0], Is.EqualTo ((byte) MDPCommand.Ready));
Assert.That (msg[4].ConvertToString (), Is.EqualTo ("test"));
// tell worker to stop gracefully
var reply = new NetMQMessage ();
reply.Push (new[] { (byte) MDPCommand.Kill });
// push MDP Version
reply.Push (msg[2]);
// push separator
reply.Push (NetMQFrame.Empty);
// push worker address
reply.Push (msg[0]);
// send reply which is a request for the worker
e.Socket.SendMessage (reply);
};
poller.AddSocket (broker);
Task.Factory.StartNew (poller.PollTillCancelled);
// set the event handler to receive the logging messages
session.LogInfoReady += (s, e) => loggingMessages.Add (e.Info);
// initialise the worker - broker protocol
session.Receive (null);
poller.CancelAndJoin ();
poller.RemoveSocket (broker);
Assert.That (loggingMessages.Count, Is.EqualTo (5));
Assert.That (loggingMessages[0], Is.EqualTo ("[WORKER] connected to broker at tcp://localhost:5557"));
Assert.That (loggingMessages[1].Contains ("[WORKER] sending"), Is.True);
Assert.That (loggingMessages[2].Contains ("[WORKER] received"));
Assert.That (loggingMessages[4].Contains ("abandoning"));
}
}
public void Request_ValidRequestStringGeneric_ShouldReturnExpectedGuid ()
{
var expectedId = Guid.NewGuid ();
var replyMessage = new NetMQMessage ();
replyMessage.Push (expectedId.ToString ());
replyMessage.Push (TitanicReturnCode.Ok.ToString ());
var fakeMDPClient = new MDPTestClientForTitanicClient { ReplyMessage = replyMessage, RequestId = expectedId };
var generic = new TestEntity ();
var sut = new TitanicClient (fakeMDPClient);
var id = sut.Request ("echo", generic.ConvertToBytes ());
id.Should ().Be (expectedId.ToString ());
}
public void Request_ValidRequestStringBytes_ShouldReturnExpectedGuid ()
{
var expectedId = Guid.NewGuid ();
var replyMessage = new NetMQMessage ();
replyMessage.Push (expectedId.ToString ());
replyMessage.Push (TitanicReturnCode.Ok.ToString ());
var fakeMDPClient = new MDPTestClientForTitanicClient { ReplyMessage = replyMessage, RequestId = expectedId };
var sut = new TitanicClient (fakeMDPClient);
var id = sut.Request ("echo", Encoding.UTF8.GetBytes ("Hello World!"));
id.Should ().Be (expectedId.ToString ());
}
public NetMQMessage Send (string serviceName, NetMQMessage request)
{
// first call is [mmi.service][servicename]
// return is [Ok]
// second call is [service][request]
// return is [reply]
m_count++;
if (m_count == 1)
{
// proceed only if commanded to -> is automatically called by TitanicBroker.Run
// and askes for a reply from a service, so wait until we want an answer
waitHandle.WaitOne ();
var reply = new NetMQMessage ();
reply.Push (MmiCode.Ok.ToString ());
return reply;
}
// wait to proceed until signaled
waitHandle.WaitOne ();
return request; // as echo service :-)
}
public Task Send(ArraySegment<byte> data, params object[] connectionIDs)
{
var task = new Task(() =>
{
var msg = new NetMQMessage();
if (_socket is RouterSocket)
{
msg.Append(new byte[0]);
msg.AppendEmptyFrame();
}
msg.Append(data.Count == data.Array.Length ? data.Array : data.ToArray());
if (connectionIDs.Length <= 0)
_socket.SendMultipartMessage(msg);
else
{
foreach (var connection in connectionIDs)
{
if (_socket is RouterSocket && connection is byte[])
{
msg.Pop();
msg.Push(((byte[])connection));
}
_socket.SendMultipartMessage(msg);
}
}
});
task.Start(_scheduler);
return task;
}
public void Run()
{
var rnd = new Random(m_id);
using (var context = NetMQContext.Create())
using (var worker = context.CreateRequestSocket())
{
worker.Connect(m_localBackendAddress);
Console.WriteLine("[WORKER {0}] Connected & READY", m_id);
// build READY message
var msg = new NetMQMessage();
var ready = NetMQFrame.Copy(new[] { Program.WorkerReady });
msg.Append(ready);
msg.Push(NetMQFrame.Empty);
msg.Push(new[] { m_id });
// and send to broker
worker.SendMessage(msg);
while (true)
{
// wait for a request - the REQ might be from a local client or a cloud request
var request = worker.ReceiveMessage();
if (request.FrameCount < 3)
{
Console.WriteLine("[WORKER {0}] ERR - received an empty message", m_id);
break; // something went wrong -> exit
}
Console.WriteLine("[WORKER {0}] received", m_id);
foreach (var frame in request)
Console.WriteLine("\t[{0}", frame.ConvertToString());
// simulate working for an arbitrary time < 2s
Thread.Sleep(rnd.Next(2000));
// simply send back what we received
worker.SendMessage(request);
}
}
}
public void CloseRequest_MaxEntryClosedAndAdditionalRequestsAndReplies_ShouldReorganizeQueue()
{
const int max_entries = 20;
const int additional_requests = 5;
var path = Path.Combine (Path.GetTempPath (), ".titanic", "Close_4");
var sut = new TitanicFileIO (path, max_entries);
var titanicQueue = sut.TitanicQueue;
for (var i = 0; i < max_entries + additional_requests; i++)
{
var id = Guid.NewGuid ();
sut.SaveNewRequestEntry (id); // -> fill titanic.queue
var message = new NetMQMessage ();
message.Push (string.Format ("Message #{0}", i));
message.Push ("echo");
sut.SaveMessage (TitanicOperation.Request, id, message);
}
foreach (var entry in sut.GetRequestEntries (null).Skip (3).Take (5))
{
sut.SaveProcessedRequestEntry (entry);
var message = sut.GetMessage (TitanicOperation.Request, entry.RequestId);
sut.SaveMessage (TitanicOperation.Reply, entry.RequestId, message);
}
var requests = sut.GetRequestEntries (null).ToArray ();
requests.Length.Should ().Be (max_entries + additional_requests);
requests.Count (re => re.State == RequestEntry.Is_Processed).Should ().Be (5);
for (var i = 0; i < max_entries; i++)
sut.CloseRequest (requests[i].RequestId); // mark closed not worrying about state
sut.GetNotClosedRequestEntries ()
.Count ()
.Should ()
.Be (additional_requests, "because 5 requests should have been left over!");
Directory.Delete (sut.TitanicDirectory, true);
}
public void ExistsMessage_ExistingMessageWrongState_ShouldUpdateCorrectRequestEntry()
{
const int id_to_retrieve = 7;
var sut = new TitanicMemoryIO ();
var ids = new Guid[10];
for (var i = 0; i < 10; i++)
{
ids[i] = Guid.NewGuid ();
var request = new NetMQMessage ();
request.Push (string.Format ("Request #{0}", i));
request.Push ("echo");
sut.SaveMessage (TitanicOperation.Request, ids[i], request);
}
sut.ExistsMessage (TitanicOperation.Reply, ids[id_to_retrieve]).Should ().BeFalse ();
}
// messages can be:
//
// REQUEST:
// in goes -> [service name][request]
// returns -> [return code][Guid]
// REPLY
// in goes -> [request id]
// returns -> [return code][reply]
// CLOSE
// in goes -> [request id]
//
public NetMQMessage Send (string serviceName, NetMQMessage message)
{
Log ($"requested service <{serviceName}> with request <{message}>");
if (ReplyMessage != null)
return new NetMQMessage (ReplyMessage); // to keep it intact for multiple tries return a copy(!)
var operation = (TitanicOperation) Enum.Parse (typeof (TitanicOperation), serviceName);
var reply = new NetMQMessage ();
switch (operation)
{
case TitanicOperation.Request:
var id = RequestId == Guid.Empty ? Guid.NewGuid () : RequestId;
reply.Push (id.ToString ());
reply.Push (TitanicReturnCode.Ok.ToString ());
break;
case TitanicOperation.Reply:
if (ReplyMessage == null)
{
reply.Push (ReplyDataFrame);
reply.Push (TitanicReturnCode.Ok.ToString ());
}
else
reply = ReplyMessage;
break;
case TitanicOperation.Close:
reply.Push (TitanicReturnCode.Ok.ToString ());
break;
default:
reply.Push (TitanicReturnCode.Failure.ToString ());
break;
}
Log ($"reply <{reply}>");
return reply;
}
public void SaveNewRequest_GuidAndRequest_ShouldUpdateQueue ()
{
var sut = new TitanicMemoryIO ();
var request = new NetMQMessage ();
request.Push ("A Request");
var id = Guid.NewGuid ();
var entry = new RequestEntry { RequestId = id, Request = request };
sut.SaveRequestEntry (entry);
var result = sut.GetRequestEntry (id);
sut.NumberOfRequests.Should ().Be (1, "because we just added one");
result.RequestId.Should ().Be (id);
result.Request.ShouldBeEquivalentTo (request);
result.Position.Should ().Be (-1);
result.State.Should ().Be (RequestEntry.Is_Pending);
}
public void PushMessage()
{
var message = new NetMQMessage();
Assert.AreEqual(0, message.FrameCount);
Assert.True(message.IsEmpty);
message.Append("Hello");
message.Push("Hello2");
Assert.AreEqual("Hello", message[1].ConvertToString());
Assert.AreEqual("Hello2", message[0].ConvertToString());
Assert.False(message.IsEmpty);
Assert.AreSame(message[0], message.First);
Assert.AreSame(message[1], message.Last);
Assert.AreNotSame(message[0], message[1]);
Assert.AreEqual(2, message.FrameCount);
}
public void PushMessage()
{
var message = new NetMQMessage();
Assert.Equal(0, message.FrameCount);
Assert.True(message.IsEmpty);
message.Append("Hello");
message.Push("Hello2");
Assert.Equal("Hello", message[1].ConvertToString());
Assert.Equal("Hello2", message[0].ConvertToString());
Assert.False(message.IsEmpty);
Assert.Same(message[0], message.First);
Assert.Same(message[1], message.Last);
Assert.NotSame(message[0], message[1]);
Assert.Equal(2, message.FrameCount);
}
public void SaveProcessedRequest_ExistingRequestWithRequestData_ShouldUpdateEntryAppropriate()
{
var sut = new TitanicMemoryIO();
var request = new NetMQMessage();
request.Push("Processed Request Data");
var id = Guid.NewGuid();
var entry = new RequestEntry {
RequestId = id, Request = request
};
sut.SaveRequestEntry(entry);
sut.SaveProcessedRequestEntry(entry);
var result = sut.GetRequestEntry(id);
result.RequestId.Should().Be(id);
result.Request.ShouldBeEquivalentTo(request);
result.Position.Should().Be(-1);
result.State.Should().Be(RequestEntry.Is_Processed);
}
/// <summary>
/// Return a new NetMQMessage that holds two frames:
/// 1. a frame with a single byte representing the HandshakeType, which is Certificate,
/// 2. a frame containing the certificate that has been exported to a byte-array.
/// </summary>
/// <returns>the resulting new NetMQMessage</returns>
public override NetMQMessage ToNetMQMessage()
{
NetMQMessage message = AddHandShakeType();
byte[] certBytes = Certificate.Export(X509ContentType.Cert);
//加长度,证书总长度
var certsLengthBytes = BitConverter.GetBytes(certBytes.Length + 3);
message.Append(new byte[] { certsLengthBytes[2], certsLengthBytes[1], certsLengthBytes[0] });
//每个证书的长度和证书
var certLengthBytes = BitConverter.GetBytes(certBytes.Length);
message.Append(new byte[] { certLengthBytes[2], certLengthBytes[1], certLengthBytes[0] });
message.Append(certBytes);
var handShakeType = message.Pop();
InsertLength(message);
message.Push(handShakeType);
return(message);
}
public void SaveNewRequest_GuidAndRequest_ShouldUpdateQueue()
{
var sut = new TitanicMemoryIO();
var request = new NetMQMessage();
request.Push("A Request");
var id = Guid.NewGuid();
var entry = new RequestEntry {
RequestId = id, Request = request
};
sut.SaveRequestEntry(entry);
var result = sut.GetRequestEntry(id);
sut.NumberOfRequests.Should().Be(1, "because we just added one");
result.RequestId.Should().Be(id);
result.Request.ShouldBeEquivalentTo(request);
result.Position.Should().Be(-1);
result.State.Should().Be(RequestEntry.Is_Pending);
}
/// <summary>
/// usage: MDPServiceDiscoveryClientExample [-v]
///
/// implements a MDPClient API usage with Service Discovery
/// </summary>
static void Main(string[] args)
{
const string service_to_lookup = "echo";
const string service_discovery = "mmi.service";
var verbose = args.Length == 1 && args[0] == "-v";
var id = Encoding.ASCII.GetBytes("SDC01");
// give WORKER & BROKER time to settle
Thread.Sleep(250);
using (var session = new MDPClient("tcp://localhost:5555", id))
{
if (verbose)
{
session.LogInfoReady += (s, e) => Console.WriteLine("{0}", e.Info);
}
var request = new NetMQMessage();
// set the service name
request.Push(service_to_lookup);
// send the request to service discovery
var reply = session.Send(service_discovery, request);
if (reply != null && !reply.IsEmpty)
{
var answer = reply.First.ConvertToString();
Console.WriteLine("Lookup {0} service returned: {1}/{2}", service_to_lookup, answer, reply);
}
else
{
Console.WriteLine("ERROR: no response from broker, seems like broker is NOT running!");
}
}
Console.Write("Exit with any key.");
Console.ReadKey();
}
/// <summary>
/// Casts the message to <see cref="NetMQMessage"/> with given <paramref name="key"/>,
/// <paramref name="peer"/> and <paramref name="version"/>.
/// </summary>
/// <param name="key">A <see cref="PrivateKey"/> to sign message.</param>
/// <param name="peer"><see cref="Peer"/>-typed representation of the
/// sender's transport layer.
/// <seealso cref="ITransport.AsPeer"/></param>
/// <param name="timestamp">The <see cref="DateTimeOffset"/> of the message is created.
/// </param>
/// <param name="version"><see cref="AppProtocolVersion"/>-typed version of the
/// transport layer.</param>
/// <returns>A <see cref="NetMQMessage"/> containing the signed <see cref="Message"/>.
/// </returns>
/// <exception cref="ArgumentNullException">Thrown when <paramref name="peer"/> is
/// <c>null</c>.</exception>
public NetMQMessage ToNetMQMessage(
PrivateKey key,
Peer peer,
DateTimeOffset timestamp,
AppProtocolVersion version)
{
if (peer is null)
{
throw new ArgumentNullException(nameof(peer));
}
var message = new NetMQMessage();
// Write body (by concrete class)
foreach (NetMQFrame frame in DataFrames)
{
message.Append(frame);
}
// Write headers. (inverse order)
message.Push(timestamp.Ticks);
message.Push(SerializePeer(peer));
message.Push((byte)Type);
message.Push(version.Token);
// Make and insert signature
byte[] signature = key.Sign(message.ToByteArray());
List <NetMQFrame> frames = message.ToList();
frames.Insert((int)MessageFrame.Sign, new NetMQFrame(signature));
message = new NetMQMessage(frames);
if (Identity is byte[] to)
{
message.Push(to);
}
return(message);
}
public void TitanicReply_RequestReplyPending_ShouldSentCorrectReply()
{
var io = new TitanicMemoryIO();
using (var reqWorker = new FakeRequestMDPWorker())
using (var repWorker = new FakeReplyMDPWorker())
using (var closeWorker = new FakeCloseMDPWorker())
using (var dispatchClient = new FakeDispatchMDPClient())
using (var sut = new TitanicBroker(io))
{
// setup the queue with a request
var guid = Guid.NewGuid();
var msg = new NetMQMessage();
msg.Push(guid.ToString());
// queue is setup with a request pending
io.SaveNewRequestEntry(guid, msg);
// setup the fake replyWorker's request
repWorker.Request = new NetMQMessage();
repWorker.Request.Push(guid.ToString());
// start the process chain - worker & client should only run until they hit an AutoResetEvent
Task.Factory.StartNew(() => sut.Run(reqWorker, repWorker, closeWorker, dispatchClient));
// signal worker to go ahead
repWorker.waitHandle.Set();
// give everything some time to process
Thread.Sleep(_sleep_for);
// TEST COMMUNICATION
repWorker.Reply.FrameCount.Should().Be(1, "because a 1 frame message is expected. ({0})", repWorker.Reply);
repWorker.Reply.First.ConvertToString().Should().Be("Pending");
// TEST QUEUE
io.GetRequestEntries(e => e.RequestId == guid).Count().Should().Be(1);
var queueEntry = io.GetRequestEntry(guid);
queueEntry.State.Should().Be(RequestEntry.Is_Pending);
io.ExistsMessage(TitanicOperation.Request, guid).Should().BeTrue();
io.ExistsMessage(TitanicOperation.Reply, guid).Should().BeFalse();
io.ExistsMessage(TitanicOperation.Close, guid).Should().BeFalse();
}
}
// messages can be:
//
// REQUEST:
// in goes -> [service name][request]
// returns -> [return code][Guid]
// REPLY
// in goes -> [request id]
// returns -> [return code][reply]
// CLOSE
// in goes -> [request id]
//
public NetMQMessage Send(string serviceName, NetMQMessage message)
{
Log(string.Format("requested service <{0}> with request <{1}>", serviceName, message));
if (ReplyMessage != null)
{
return(new NetMQMessage(ReplyMessage)); // to keep it intact for multiple tries return a copy(!)
}
var operation = (TitanicOperation)Enum.Parse(typeof(TitanicOperation), serviceName);
var reply = new NetMQMessage();
switch (operation)
{
case TitanicOperation.Request:
var id = RequestId == Guid.Empty ? Guid.NewGuid() : RequestId;
reply.Push(id.ToString());
reply.Push(TitanicReturnCode.Ok.ToString());
break;
case TitanicOperation.Reply:
if (ReplyMessage == null)
{
reply.Push(ReplyDataFrame);
reply.Push(TitanicReturnCode.Ok.ToString());
}
else
{
reply = ReplyMessage;
}
break;
case TitanicOperation.Close:
reply.Push(TitanicReturnCode.Ok.ToString());
break;
default:
reply.Push(TitanicReturnCode.Failure.ToString());
break;
}
Log(string.Format("reply <{0}>", reply));
return(reply);
}
public void DifferentAppProtocolVersionWithDifferentStructure()
{
var validAppProtocolVersion = new AppProtocolVersion(
1,
new Bencodex.Types.Integer(0),
ImmutableArray <byte> .Empty,
default(Address));
var invalidAppProtocolVersion = new AppProtocolVersion(
2,
new Bencodex.Types.Integer(0),
ImmutableArray <byte> .Empty,
default(Address));
var netMQMessage = new NetMQMessage();
netMQMessage.Push(validAppProtocolVersion.Token);
Assert.Throws <DifferentAppProtocolVersionException>(() =>
Message.Parse(
netMQMessage,
true,
invalidAppProtocolVersion,
ImmutableHashSet <PublicKey> .Empty,
null));
}
private void MultipleRequestClient(string serviceName, string endpoint, IMDPClient client = null)
{
const int _NO_OF_RUNS = 100;
var reply = new NetMQMessage[_NO_OF_RUNS];
var idC01 = new[] { (byte)'C', (byte)'1' };
client = client ?? new MDPClient(endpoint, idC01);
var request = new NetMQMessage();
// set the request data
request.Push("Helo World!");
// send the request to the service
for (int i = 0; i < _NO_OF_RUNS; i++)
{
reply[i] = client.Send(serviceName, request);
}
client.Dispose();
Assert.That(reply, Has.None.Null);
Assert.That(reply.All(r => r.First.ConvertToString() == "Helo World!"));
}
/// <summary>
/// usage: MDPServiceDiscoveryClientExample [-v]
///
/// implements a MDPClient API usage with Service Discovery
/// </summary>
private static void Main (string[] args)
{
const string service_to_lookup = "echo";
const string service_discovery = "mmi.service";
var verbose = args.Length == 1 && args[0] == "-v";
var id = Encoding.ASCII.GetBytes ("SDC01");
// give WORKER & BROKER time to settle
Thread.Sleep (250);
using (var session = new MDPClient ("tcp://localhost:5555", id))
{
if (verbose)
session.LogInfoReady += (s, e) => Console.WriteLine ("{0}", e.Info);
var request = new NetMQMessage ();
// set the service name
request.Push (service_to_lookup);
// send the request to service discovery
var reply = session.Send (service_discovery, request);
if (reply != null && !reply.IsEmpty)
{
var answer = reply.First.ConvertToString ();
Console.WriteLine ("Lookup {0} service returned: {1}/{2}", service_to_lookup, answer, reply);
}
else
Console.WriteLine ("ERROR: no response from broker, seems like broker is NOT running!");
}
Console.Write ("Exit with any key.");
Console.ReadKey ();
}
public void TitanicReply_RequestReplyPending_ShouldSentCorrectReply ()
{
var io = new TitanicMemoryIO ();
using (var reqWorker = new FakeRequestMDPWorker ())
using (var repWorker = new FakeReplyMDPWorker ())
using (var closeWorker = new FakeCloseMDPWorker ())
using (var dispatchClient = new FakeDispatchMDPClient ())
using (var sut = new TitanicBroker (io))
{
// setup the queue with a request
var guid = Guid.NewGuid ();
var msg = new NetMQMessage ();
msg.Push (guid.ToString ());
// queue is setup with a request pending
io.SaveNewRequestEntry (guid, msg);
// setup the fake replyWorker's request
repWorker.Request = new NetMQMessage ();
repWorker.Request.Push (guid.ToString ());
// start the process chain - worker & client should only run until they hit an AutoResetEvent
Task.Factory.StartNew (() => sut.Run (reqWorker, repWorker, closeWorker, dispatchClient));
// signal worker to go ahead
repWorker.waitHandle.Set ();
// give everything some time to process
Thread.Sleep (_sleep_for);
// TEST COMMUNICATION
repWorker.Reply.FrameCount.Should ().Be (1, "because a 1 frame message is expected. ({0})", repWorker.Reply);
repWorker.Reply.First.ConvertToString ().Should ().Be ("Pending");
// TEST QUEUE
io.GetRequestEntries (e => e.RequestId == guid).Count ().Should ().Be (1);
var queueEntry = io.GetRequestEntry (guid);
queueEntry.State.Should ().Be (RequestEntry.Is_Pending);
io.ExistsMessage (TitanicOperation.Request, guid).Should ().BeTrue ();
io.ExistsMessage (TitanicOperation.Reply, guid).Should ().BeFalse ();
io.ExistsMessage (TitanicOperation.Close, guid).Should ().BeFalse ();
}
}
public override void SendMessageToOwner(NetMQMessage message)
{
message.Push(Endpoint.Commands.PLUG_MSG.ToString()); //Inject message type for whisper to seperate
Whisper(Uuid, message);
}
/// <summary>
/// ParanoidPirate.Queue [-v]
///
/// Does load-balancing with heartbeating on worker tasks to detect
/// crashed, blocked or slow running worker tasks .
/// </summary>
private static void Main(string[] args)
{
Console.Title = "NetMQ ParanoidPirate Queue";
// serves as flag for exiting the program
var exit = false;
// catch CTRL+C as exit command
Console.CancelKeyPress += (s, e) =>
{
e.Cancel = true;
exit = true;
};
var verbose = args.Length > 0 && args[0] == "-v";
using (var frontend = new RouterSocket())
using (var backend = new RouterSocket())
using (var poller = new NetMQPoller())
{
frontend.Bind(Commons.QueueFrontend);
backend.Bind(Commons.QueueBackend);
var workers = new Workers();
// client sends to this socket
frontend.ReceiveReady += (s, e) =>
{
// only process incoming client requests
// if we have workers available handle client requests as long as we have workers
// storage capability of the socket otherwise and pick up later
if (workers.Available)
{
// get all message frames!
var request = frontend.ReceiveMultipartMessage();
if (verbose)
Console.WriteLine("[QUEUE] received {0}", request);
// get next available worker
var worker = workers.Next();
// wrap message with worker's address
var msg = Wrap(worker, request);
if (verbose)
Console.WriteLine("[QUEUE -> WORKER] sending {0}", msg);
backend.SendMultipartMessage(msg);
}
};
// worker sends to this socket
backend.ReceiveReady += (s, e) =>
{
var msg = e.Socket.ReceiveMultipartMessage();
if (verbose)
Console.WriteLine("[QUEUE <- WORKER] received {0}", msg);
// use workers identity for load-balancing
var workerIdentity = Unwrap(msg);
var worker = new Worker(workerIdentity);
workers.Ready(worker);
// just convenience
var readableWorkerId = workerIdentity.ConvertToString();
if (msg.FrameCount == 1)
{
var data = msg[0].ConvertToString();
// the message is either READY or HEARTBEAT or corrupted
switch (data)
{
case Commons.PPPHeartbeat:
Console.WriteLine("[QUEUE <- WORKER] Received a Heartbeat from {0}",
readableWorkerId);
break;
case Commons.PPPReady:
Console.WriteLine("[QUEUE <- WORKER] Received a READY form {0}",
readableWorkerId);
break;
default:
Console.WriteLine("[QUEUE <- WORKER] ERROR received an invalid message!");
break;
}
}
else
{
if (verbose)
Console.WriteLine("[QUEUE -> CLIENT] sending {0}", msg);
frontend.SendMultipartMessage(msg);
}
};
var timer = new NetMQTimer(Commons.HeartbeatInterval);
// every specified ms QUEUE shall send a heartbeat to all connected workers
timer.Elapsed += (s, e) =>
{
// send heartbeat to every worker
foreach (var worker in workers)
{
var heartbeat = new NetMQMessage();
heartbeat.Push(new NetMQFrame(Commons.PPPHeartbeat));
heartbeat.Push(worker.Identity);
Console.WriteLine("[QUEUE -> WORKER] sending heartbeat!");
backend.SendMultipartMessage(heartbeat);
}
// restart timer
e.Timer.Enable = true;
// remove all dead or expired workers
workers.Purge();
};
if (verbose)
Console.WriteLine("[QUEUE] Start listening!");
poller.Add(frontend);
poller.Add(backend);
poller.RunAsync();
// hit CRTL+C to stop the while loop
while (!exit)
Thread.Sleep(100);
}
}
/// <summary>
/// Prepend the message with an empty frame as separator and a frame
/// </summary>
/// <returns>new message with wrapped content</returns>
private NetMQMessage Wrap(NetMQMessage msg, NetMQFrame frame)
{
var result = new NetMQMessage(msg);
result.Push(NetMQFrame.Empty); // according to MDP an empty frame is the separator
result.Push(frame); // the return address
return result;
}
private void EchoClient(IMDPClient client, string serviceName)
{
var request = new NetMQMessage();
// set the request data
request.Push("Helo World!");
// send the request to the service
var reply = client.Send(serviceName, request);
Assert.That(reply, Is.Not.Null, "[ECHO CLIENT] REPLY was <null>");
Assert.That(reply.FrameCount, Is.EqualTo(1));
Assert.That(reply.First.ConvertToString(), Is.EqualTo("Helo World!"));
}
public void Receive_REPLYtoREQUEST_ShouldSendCorrectReply()
{
const string hostAddress = "tcp://localhost:5557";
var loggingMessages = new List <string> ();
// setup the counter socket for communication
using (var context = NetMQContext.Create())
using (var broker = context.CreateRouterSocket())
using (var poller = new Poller())
using (var session = new MDPWorker(hostAddress, "test", new[] { (byte)'W', (byte)'1' }))
{
broker.Bind(hostAddress);
// we need to pick up any message in order to avoid errors
broker.ReceiveReady += (s, e) =>
{
var msg = e.Socket.ReceiveMultipartMessage();
if (msg[3].Buffer[0] == (byte)MDPCommand.Ready)
{
// this is a READY message and we
// send REQUEST message
var request = new NetMQMessage();
request.Push("echo test"); // [request]
request.Push(NetMQFrame.Empty); // [e][request]
request.Push("C1"); // [client adr][e][request]
request.Push(new[] { (byte)MDPCommand.Request }); // [command][client adr][e][request]
request.Push(msg[2]); // [header][command][client adr][e][request]
request.Push(NetMQFrame.Empty); // [e][header][command][client adr][e][request]
request.Push(msg[0]); // [worker adr][e][header][command][client adr][e][request]
// send reply which is a request for the worker
e.Socket.SendMessage(request);
}
if (msg[3].Buffer[0] == (byte)MDPCommand.Reply)
{
// we expect to receive
// [WORKER ADR][e]["MDPW01"][REPLY][CLIENT ADR][e][request == "echo test"]
// make sure the frames are as expected
Assert.That(msg[0].ConvertToString(), Is.EqualTo("W1"));
Assert.That(msg[1], Is.EqualTo(NetMQFrame.Empty));
Assert.That(msg[2].ConvertToString(), Is.EqualTo("MDPW01"));
Assert.That(msg[3].BufferSize, Is.EqualTo(1));
Assert.That(msg[3].Buffer[0], Is.EqualTo((byte)MDPCommand.Reply));
Assert.That(msg[4].ConvertToString(), Is.EqualTo("C1"));
Assert.That(msg[5], Is.EqualTo(NetMQFrame.Empty));
Assert.That(msg[6].ConvertToString(), Is.EqualTo("echo test"));
// tell worker to stop gracefully
var reply = new NetMQMessage();
reply.Push(new[] { (byte)MDPCommand.Kill });
// push MDP Version
reply.Push(msg[2]);
// push separator
reply.Push(NetMQFrame.Empty);
// push worker address
reply.Push(msg[0]);
// send reply which is a request for the worker
e.Socket.SendMessage(reply);
}
};
poller.AddSocket(broker);
Task.Factory.StartNew(poller.PollTillCancelled);
// set the event handler to receive the logging messages
session.LogInfoReady += (s, e) => loggingMessages.Add(e.Info);
// initialise the worker - broker protocol
// and get initial request
var workerRequest = session.Receive(null);
// just echo the request
session.Receive(workerRequest);
poller.CancelAndJoin();
poller.RemoveSocket(broker);
Assert.That(loggingMessages.Count, Is.EqualTo(8));
Assert.That(loggingMessages[0], Is.EqualTo("[WORKER] connected to broker at tcp://localhost:5557"));
Assert.That(loggingMessages[1].Contains("Ready"));
Assert.That(loggingMessages[2].Contains("[WORKER] received"));
Assert.That(loggingMessages[3].Contains("Request"));
Assert.That(loggingMessages[4].Contains("Reply"));
Assert.That(loggingMessages[6].Contains("Kill"));
Assert.That(loggingMessages[7].Contains("abandoning"));
}
}
protected override void OnIncomingMessage(byte[] identity, NetMQMessage message)
{
message.Push(identity);
WriteIngoing(message);
}
/// <summary>
/// wraps the message with the identity and includes an empty frame
/// </summary>
/// <returns>[socket.Identity][empty][old message]</returns>
private static NetMQMessage Wrap(byte[] identity, NetMQMessage msg)
{
var result = new NetMQMessage(msg);
result.Push(NetMQFrame.Empty);
result.Push(identity);
return result;
}
private void AddHelloClient(IMDPClient client, string serviceName)
{
const string _PAYLOAD = "Hello World";
var request = new NetMQMessage();
// set the request data
request.Push(_PAYLOAD);
// send the request to the service
var reply = client.Send(serviceName, request);
Assert.That(reply, Is.Not.Null, "[ADD HELLO CLIENT] REPLY was <null>");
Assert.That(reply.FrameCount, Is.EqualTo(1));
Assert.That(reply.First.ConvertToString(), Is.EqualTo(_PAYLOAD + " - HELLO"));
}
public void Run()
{
Console.WriteLine("[CLIENT {0}] Starting", m_id);
var rnd = new Random(m_id);
// each request shall have an unique id in order to recognize an reply for a request
var messageId = new byte[5];
// create clientId for messages
var clientId = new[] { m_id };
// a flag to signal that an answer has arrived
bool messageAnswered = false;
// we use a poller because we have a socket and a timer to monitor
using (var clientPoller = new Poller())
using (var context = NetMQContext.Create())
using (var client = context.CreateRequestSocket())
using (var monitor = context.CreatePushSocket())
{
client.Connect(m_localFrontendAddress);
monitor.Connect(m_monitorAddress);
client.Options.Identity = new[] { m_id };
var timer = new NetMQTimer((int)TimeSpan.FromSeconds(10).TotalMilliseconds);
// use as flag to indicate exit
var exit = false;
// every 10 s check if message has been received, if not then send error message and ext
// and restart timer otherwise
timer.Elapsed += (s, e) =>
{
if (messageAnswered)
{
e.Timer.Enable = true;
}
else
{
var msg = string.Format("[CLIENT {0}] ERR - EXIT - lost message {1}", m_id, messageId);
// send an error message
monitor.Send(msg);
// if poller is started than stop it
if (clientPoller.IsStarted)
clientPoller.CancelAndJoin();
// mark the required exit
exit = true;
}
};
// process arriving answers
client.ReceiveReady += (s, e) =>
{
// mark the arrival of an answer
messageAnswered = true;
// worker is supposed to answer with our request id
var reply = e.Socket.ReceiveMultipartMessage();
if (reply.FrameCount == 0)
{
// something went wrong
monitor.Send(string.Format("[CLIENT {0}] Received an empty message!", m_id));
// mark the exit flag to ensure the exit
exit = true;
}
else
{
var sb = new StringBuilder();
// create success message
foreach (var frame in reply)
sb.Append("[" + frame.ConvertToString() + "]");
// send the success message
monitor.Send(string.Format("[CLIENT {0}] Received answer {1}",
m_id,
sb.ToString()));
}
};
// add socket & timer to poller
clientPoller.AddSocket(client);
clientPoller.AddTimer(timer);
// start poller in another thread to allow the continued processing
clientPoller.PollTillCancelledNonBlocking();
// if true the message has been answered
// the 0th message is always answered
messageAnswered = true;
while (!exit)
{
// simulate sporadic activity by randomly delaying
Thread.Sleep((int)TimeSpan.FromSeconds(rnd.Next(5)).TotalMilliseconds);
// only send next message if the previous one has been replied to
if (messageAnswered)
{
// generate random 5 byte as identity for for the message
rnd.NextBytes(messageId);
messageAnswered = false;
// create message [client adr][empty][message id] and send it
var msg = new NetMQMessage();
msg.Push(messageId);
msg.Push(NetMQFrame.Empty);
msg.Push(clientId);
client.SendMessage(msg);
}
}
// stop poller if needed
if (clientPoller.IsStarted)
clientPoller.CancelAndJoin();
}
}
/// <summary>
/// usage: MDPClientExample [-v] [-rn] (1 ;lt n ;lt 100000 / Default == 10)
///
/// implements a MDPClient API usage
/// </summary>
private static void Main (string[] args)
{
if (args.Length == 1 || args.Length > 2)
{
if (!(args[0] == "-v" || args[0].Contains ("-r")))
{
Console.WriteLine ("MDPClientExample [-v(erbose) OR -h(elp)]");
Console.WriteLine ("\t-v => verbose");
Console.WriteLine ("\tto stop processing use CTRL+C");
return;
}
}
const string service_name = "echo";
const int max_runs = 100000;
bool verbose = false;
int runs = 10;
if (args.Length >= 1)
{
if (args.Length == 1 && args[0] == "-v")
verbose = true;
else if (args.Length == 2)
verbose = args[0] == "-v" || args[1] == "-v";
if (args[0].Contains ("-r") || args.Length > 1)
{
if (args[0].Contains ("-r"))
runs = GetInt (args[0]);
else if (args[1].Contains ("-r"))
runs = GetInt (args[1]);
}
}
runs = runs == -1 ? 10 : runs > max_runs ? max_runs : runs;
var id = new[] { (byte) 'C', (byte) '1' };
var watch = new Stopwatch ();
Console.WriteLine ("Starting MDPClient and will send {0} requests to service <{1}>.", runs, service_name);
Console.WriteLine("(writes '.' for every 100 and '|' for every 1000 requests)\n");
try
{
// create MDP client and set verboseness && use automatic disposal
using (var session = new MDPClient ("tcp://localhost:5555", id))
{
if (verbose)
session.LogInfoReady += (s, e) => Console.WriteLine ("{0}", e.Info);
// just give everything time to settle in
Thread.Sleep (500);
watch.Start ();
for (int count = 0; count < runs; count++)
{
var request = new NetMQMessage ();
// set the request data
request.Push ("Helo World!");
// send the request to the service
var reply = session.Send (service_name, request);
if (ReferenceEquals (reply, null))
break;
if (count % 1000 == 0)
Console.Write ("|");
else
if (count % 100 == 0)
Console.Write (".");
}
watch.Stop ();
}
}
catch (Exception ex)
{
Console.WriteLine ("ERROR:");
Console.WriteLine (ex.Message);
Console.WriteLine (ex.StackTrace);
return;
}
var time = watch.Elapsed;
Console.WriteLine ("{0} request/replies in {1} ms processed! Took {2:N3} ms per REQ/REP",
runs,
time.TotalMilliseconds,
time.TotalMilliseconds / runs);
Console.Write ("\nExit with any key!");
Console.ReadKey ();
}
/// <summary>
/// send a request to a broker for a specific service and receive the reply
///
/// if the reply is not available within a specified time
/// the client deems the connection as lost and reconnects
/// for a specified number of times. if no reply is available
/// throughout this procedure the client abandons
/// the reply is checked for validity and returns the reply
/// according to the verbose flag it reports about its activities
/// </summary>
/// <param name="serviceName">the name of the service requested</param>
/// <param name="request">the request message to process by service</param>
/// <returns>the reply from service</returns>
/// <exception cref="ApplicationException">malformed message received</exception>
/// <exception cref="ApplicationException">malformed header received</exception>
/// <exception cref="ApplicationException">reply received from wrong service</exception>
public NetMQMessage Send(string serviceName, NetMQMessage request)
{
if (string.IsNullOrWhiteSpace(serviceName))
{
throw new ApplicationException("serviceName must not be empty or null.");
}
if (ReferenceEquals(request, null))
{
throw new ApplicationException("the request must not be null");
}
// memorize it for the event handler
m_serviceName = serviceName;
// if for any reason the socket is NOT connected -> connect it!
if (!m_connected)
{
Connect();
}
var message = new NetMQMessage(request);
// prefix the request according to MDP specs
// Frame 1: "MDPCxy" (six bytes MDP/Client x.y)
// Frame 2: service name as printable string
// Frame 3: request
message.Push(serviceName);
message.Push(m_mdpClient);
Log(string.Format("[CLIENT INFO] sending {0} to service {1}", message, serviceName));
var retiesLeft = Retries;
while (retiesLeft > 0)
{
// beware of an exception if broker has not picked up the message at all
// because one can not send multiple times! it is strict REQ -> REP -> REQ ...
m_client.SendMessage(message);
// Poll -> see ReceiveReady for event handling
if (m_client.Poll(Timeout))
{
// set by event handler
return(m_reply);
}
// if it failed assume communication dead and re-connect
if (--retiesLeft > 0)
{
Log("[CLIENT WARNING] no reply, reconnecting ...");
Connect();
}
}
Log("[CLIENT ERROR] permanent error, abandoning!");
m_client.Dispose();
m_ctx.Dispose();
return(null);
}
/// <summary>
/// ParanoidPirate.Queue [-v]
///
/// Does load-balancing with heartbeating on worker tasks to detect
/// crashed, blocked or slow running worker tasks .
/// </summary>
private static void Main(string[] args)
{
Console.Title = "NetMQ ParanoidPirate Queue";
// serves as flag for exiting the program
var exit = false;
// catch CTRL+C as exit command
Console.CancelKeyPress += (s, e) =>
{
e.Cancel = true;
exit = true;
};
var verbose = args.Length > 0 && args[0] == "-v";
using (var frontend = new RouterSocket())
using (var backend = new RouterSocket())
using (var poller = new NetMQPoller())
{
frontend.Bind(Commons.QueueFrontend);
backend.Bind(Commons.QueueBackend);
var workers = new Workers();
// client sends to this socket
frontend.ReceiveReady += (s, e) =>
{
// only process incoming client requests
// if we have workers available handle client requests as long as we have workers
// storage capability of the socket otherwise and pick up later
if (workers.Available)
{
// get all message frames!
var request = frontend.ReceiveMultipartMessage();
if (verbose)
{
Console.WriteLine("[QUEUE] received {0}", request);
}
// get next available worker
var worker = workers.Next();
// wrap message with worker's address
var msg = Wrap(worker, request);
if (verbose)
{
Console.WriteLine("[QUEUE -> WORKER] sending {0}", msg);
}
backend.SendMultipartMessage(msg);
}
};
// worker sends to this socket
backend.ReceiveReady += (s, e) =>
{
var msg = e.Socket.ReceiveMultipartMessage();
if (verbose)
{
Console.WriteLine("[QUEUE <- WORKER] received {0}", msg);
}
// use workers identity for load-balancing
var workerIdentity = Unwrap(msg);
var worker = new Worker(workerIdentity);
workers.Ready(worker);
// just convenience
var readableWorkerId = workerIdentity.ConvertToString();
if (msg.FrameCount == 1)
{
var data = msg[0].ConvertToString();
// the message is either READY or HEARTBEAT or corrupted
switch (data)
{
case Commons.PPPHeartbeat:
Console.WriteLine("[QUEUE <- WORKER] Received a Heartbeat from {0}",
readableWorkerId);
break;
case Commons.PPPReady:
Console.WriteLine("[QUEUE <- WORKER] Received a READY form {0}",
readableWorkerId);
break;
default:
Console.WriteLine("[QUEUE <- WORKER] ERROR received an invalid message!");
break;
}
}
else
{
if (verbose)
{
Console.WriteLine("[QUEUE -> CLIENT] sending {0}", msg);
}
frontend.SendMultipartMessage(msg);
}
};
var timer = new NetMQTimer(Commons.HeartbeatInterval);
// every specified ms QUEUE shall send a heartbeat to all connected workers
timer.Elapsed += (s, e) =>
{
// send heartbeat to every worker
foreach (var worker in workers)
{
var heartbeat = new NetMQMessage();
heartbeat.Push(new NetMQFrame(Commons.PPPHeartbeat));
heartbeat.Push(worker.Identity);
Console.WriteLine("[QUEUE -> WORKER] sending heartbeat!");
backend.SendMultipartMessage(heartbeat);
}
// restart timer
e.Timer.Enable = true;
// remove all dead or expired workers
workers.Purge();
};
if (verbose)
{
Console.WriteLine("[QUEUE] Start listening!");
}
poller.Add(frontend);
poller.Add(backend);
poller.RunAsync();
// hit CRTL+C to stop the while loop
while (!exit)
{
Thread.Sleep(100);
}
}
}
public void InsertLength(NetMQMessage message)
{
byte[] lengthBytes = new byte[3];
GetLength(lengthBytes, message);
message.Push(lengthBytes);
}
public void Receive_RequestWithWrongMDPComand_ShouldLogCorrectMessage()
{
const string hostAddress = "tcp://localhost:5555";
var loggingMessages = new List <string> ();
var first = true;
// setup the counter socket for communication
using (var context = NetMQContext.Create())
using (var broker = context.CreateRouterSocket())
using (var poller = new Poller())
using (var session = new MDPWorker(hostAddress, "test", Encoding.ASCII.GetBytes("Worker"), 2))
{
broker.Bind(hostAddress);
// we need to pick up any message in order to avoid errors
broker.ReceiveReady += (s, e) =>
{
var msg = e.Socket.ReceiveMultipartMessage();
// we expect to receive a 5 Frame message
// [WORKER ADR][EMPTY]["MDPW01"]["READY"]["test"]
if (msg.FrameCount != 5)
{
return; // it is a HEARTBEAT
}
// make sure the frames are as expected
Assert.That(msg[1], Is.EqualTo(NetMQFrame.Empty));
Assert.That(msg[2].ConvertToString(), Is.EqualTo("MDPW01"));
Assert.That(msg[3].BufferSize, Is.EqualTo(1));
Assert.That(msg[3].Buffer[0], Is.EqualTo((byte)MDPCommand.Ready));
Assert.That(msg[4].ConvertToString(), Is.EqualTo("test"));
// tell worker to stop gracefully
var reply = new NetMQMessage();
if (first)
{
reply.Push(new[] { (byte)0xff });
first = false;
}
else
{
reply.Push(new[] { (byte)MDPCommand.Kill });
}
// push MDP Version
reply.Push("MDPW01");
// push separator
reply.Push(NetMQFrame.Empty);
// push worker address
reply.Push(msg[0]);
// send reply which is a request for the worker
e.Socket.SendMessage(reply);
};
// set the event handler to receive the logging messages
session.LogInfoReady += (s, e) => loggingMessages.Add(e.Info);
poller.AddSocket(broker);
Task.Factory.StartNew(poller.PollTillCancelled);
session.HeartbeatDelay = TimeSpan.FromMilliseconds(250);
session.ReconnectDelay = TimeSpan.FromMilliseconds(250);
// initialise the worker - broker protocol
session.Receive(null);
Assert.That(loggingMessages.Count(m => m.Contains("[WORKER ERROR] invalid command received")), Is.EqualTo(1));
Assert.That(loggingMessages.Count(m => m.Contains("abandoning")), Is.EqualTo(1));
poller.CancelAndJoin();
poller.RemoveSocket(broker);
}
}
/// <summary>
/// send a asyncronous request to a broker for a specific service without receiving a reply
///
/// there is no retry logic
/// according to the verbose flag it reports about its activities
/// </summary>
/// <param name="serviceName">the name of the service requested</param>
/// <param name="request">the request message to process by service</param>
/// <exception cref="ApplicationException">serviceName must not be empty or null.</exception>
/// <exception cref="ApplicationException">the request must not be null</exception>
public void Send([NotNull] string serviceName, NetMQMessage request)
{
// TODO criar tabela de pedidos ongoing
if (string.IsNullOrWhiteSpace(serviceName))
throw new ApplicationException("serviceName must not be empty or null.");
if (ReferenceEquals(request, null))
throw new ApplicationException("the request must not be null");
// memorize it for the event handler
m_serviceName = serviceName;
// if for any reason the socket is NOT connected -> connect it!
if (!m_connected)
Connect();
var message = new NetMQMessage(request);
// prefix the request according to MDP specs
// Frame 1: Empty Frame, because DEALER socket needs to emulate a REQUEST socket to comunicate with ROUTER socket
// Frame 2: "MDPCxy" (six bytes MDP/Client x.y)
// Frame 3: service name as printable string
// Frame 4: request
message.Push(serviceName);
message.Push(m_mdpClient);
message.PushEmptyFrame();
Log($"[CLIENT INFO] sending {message} to service {serviceName}");
m_client.SendMultipartMessage(message); // Or TrySend!? ErrorHandling!?
}
public void ExistsMessage_NotExistingMessage_ShouldUpdateCorrectRequestEntry ()
{
var sut = new TitanicMemoryIO ();
var ids = new Guid[10];
for (var i = 0; i < 10; i++)
{
ids[i] = Guid.NewGuid ();
var request = new NetMQMessage ();
request.Push ($"Request #{i}");
request.Push ("echo");
sut.SaveMessage (TitanicOperation.Request, ids[i], request);
}
sut.ExistsMessage (TitanicOperation.Request, Guid.NewGuid ()).Should ().BeFalse ();
}
private void MultipleRequestClient(string serviceName, string endpoint, IMDPClient client = null)
{
const int _NO_OF_RUNS = 100;
var reply = new NetMQMessage[_NO_OF_RUNS];
var idC01 = new[] { (byte)'C', (byte)'1' };
client = client ?? new MDPClient(endpoint, idC01);
var request = new NetMQMessage();
// set the request data
request.Push("Helo World!");
// send the request to the service
for (int i = 0; i < _NO_OF_RUNS; i++)
reply[i] = client.Send(serviceName, request);
client.Dispose();
Assert.That(reply, Has.None.Null);
Assert.That(reply.All(r => r.First.ConvertToString() == "Helo World!"));
}
/// <summary>
/// process a READY, REPLY, HEARTBEAT, DISCONNECT message sent to the broker by a worker
/// </summary>
/// <param name="sender">the sender identity frame</param>
/// <param name="message">the message sent</param>
public void ProcessWorkerMessage (NetMQFrame sender, NetMQMessage message)
{
// should be
// READY [mdp command][service name]
// REPLY [mdp command][client adr][e][reply]
// HEARTBEAT [mdp command]
if (message.FrameCount < 1)
throw new ApplicationException ("Message with too few frames received.");
var mdpCommand = message.Pop (); // get the mdp command frame it should have only one byte payload
if (mdpCommand.BufferSize > 1)
throw new ApplicationException ("The MDPCommand frame had more than one byte!");
var cmd = (MDPCommand) mdpCommand.Buffer[0]; // [service name] or [client adr][e][reply]
var workerId = sender.ConvertToString (); // get the id of the worker sending the message
var workerIsKnown = m_knownWorkers.Any (w => w.Id == workerId);
switch (cmd)
{
case MDPCommand.Ready:
if (workerIsKnown)
{
// then it is not the first command in session -> WRONG
// if the worker is know to a service, remove it from that
// service and a potential waiting list therein
RemoveWorker (m_knownWorkers.Find (w => w.Id == workerId));
Log (string.Format ("[BROKER] READY out of sync. Removed worker {0}.", workerId));
}
else
{
// now a new - not know - worker sent his READY initiation message
// attach worker to service and mark as idle - worker has send READY as first message
var serviceName = message.Pop ().ConvertToString ();
var service = ServiceRequired (serviceName);
// create a new worker and set the service it belongs to
var worker = new Worker (workerId, sender, service);
// now add the worker
AddWorker (worker, service);
Log (string.Format ("[BROKER] READY processed. Worker {0} added to service {1}",
workerId,
serviceName));
}
break;
case MDPCommand.Reply:
if (workerIsKnown)
{
var worker = m_knownWorkers.Find (w => w.Id == workerId);
// remove the client return envelope and insert the protocol header
// and service name then rewrap the envelope
var client = UnWrap (message); // [reply]
message.Push (worker.Service.Name); // [service name][reply]
message.Push (MDPClientHeader); // [protocol header][service name][reply]
var reply = Wrap (client, message); // [client adr][e][protocol header][service name][reply]
Socket.SendMessage (reply);
Log (string.Format ("[BROKER] REPLY from {0} received and send to {1} -> {2}",
workerId,
client.ConvertToString (),
message));
// relist worker for further requests
AddWorker (worker, worker.Service);
}
break;
case MDPCommand.Heartbeat:
if (workerIsKnown)
{
var worker = m_knownWorkers.Find (w => w.Id == workerId);
worker.Expiry = DateTime.UtcNow + m_heartbeatExpiry;
Log (string.Format ("[BROKER] HEARTBEAT from {0} received.", workerId));
}
break;
default:
Log ("[BROKER] ERROR: Invalid MDPCommand received or message received!");
break;
}
}
/// <summary>
/// process a READY, REPLY, HEARTBEAT, DISCONNECT message sent to the broker by a worker
/// </summary>
/// <param name="sender">the sender identity frame</param>
/// <param name="message">the message sent</param>
public void ProcessWorkerMessage([NotNull] NetMQFrame sender, [NotNull] NetMQMessage message)
{
// should be
// READY [mdp command][service name]
// REPLY [mdp command][client adr][e][reply]
// HEARTBEAT [mdp command]
if (message.FrameCount < 1)
{
throw new ApplicationException("Message with too few frames received.");
}
var mdpCommand = message.Pop(); // get the mdp command frame it should have only one byte payload
if (mdpCommand.BufferSize > 1)
{
throw new ApplicationException("The MDPCommand frame had more than one byte!");
}
var cmd = (MDPCommand)mdpCommand.Buffer[0]; // [service name] or [client adr][e][reply]
var workerId = sender.ConvertToString(); // get the id of the worker sending the message
var workerIsKnown = m_knownWorkers.Any(w => w.Id == workerId);
switch (cmd)
{
case MDPCommand.Ready:
if (workerIsKnown)
{
// then it is not the first command in session -> WRONG
// if the worker is know to a service, remove it from that
// service and a potential waiting list therein
RemoveWorker(m_knownWorkers.Find(w => w.Id == workerId));
Log($"READY out of sync. Removed worker {workerId}.");
}
else
{
// now a new - not know - worker sent his READY initiation message
// attach worker to service and mark as idle - worker has send READY as first message
var serviceName = message.Pop().ConvertToString();
var service = ServiceRequired(serviceName);
// create a new worker and set the service it belongs to
var worker = new Worker(workerId, sender, service);
// now add the worker
AddWorker(worker, service);
Log($"READY processed. Worker {workerId} added to service {serviceName}");
}
break;
case MDPCommand.Reply:
if (workerIsKnown)
{
var worker = m_knownWorkers.Find(w => w.Id == workerId);
// remove the client return envelope and insert the protocol header
// and service name then rewrap the envelope
var client = UnWrap(message); // [reply]
message.Push(worker.Service.Name); // [service name][reply]
message.Push(MDPClientHeader); // [protocol header][service name][reply]
var reply = Wrap(client, message); // [client adr][e][protocol header][service name][reply]
Socket.SendMultipartMessage(reply);
DebugLog($"REPLY from {workerId} received and send to {client.ConvertToString()} -> {message}");
// relist worker for further requests
AddWorker(worker, worker.Service);
}
break;
case MDPCommand.Heartbeat:
if (workerIsKnown)
{
var worker = m_knownWorkers.Find(w => w.Id == workerId);
worker.Expiry = DateTime.UtcNow + m_heartbeatExpiry;
DebugLog($"HEARTBEAT from {workerId} received.");
}
break;
default:
Log("ERROR: Invalid MDPCommand received or message received!");
break;
}
}
/// <summary>
/// an idempotent method processing all requests to close a request with a GUID
/// it is safe to call it multiple times with the same GUID
/// </summary>
internal void ProcessTitanicClose(IMDPWorker mdpWorker = null)
{
// get a MDP worker with an automatic id and register with the service "titanic.Close"
// the worker will automatically start and connect to MDP Broker with the indicated address
using (var worker = mdpWorker ?? new MDPWorker (m_titanicAddress, TitanicOperation.Close.ToString ()))
{
NetMQMessage reply = null;
while (true)
{
// initiate the communication with sending a null, since there is no reply yet
var request = worker.Receive (reply);
Log (string.Format ("[TITANIC CLOSE] received: {0}", request));
//! has there been a breaking cause? -> exit
if (ReferenceEquals (request, null))
break;
// we expect [Guid] as the only frame
var guidAsString = request.Pop ().ConvertToString ();
var guid = Guid.Parse (guidAsString);
Log (string.Format ("[TITANIC CLOSE] closing {0}", guid));
// close the request
m_io.CloseRequest (guid);
// send back the confirmation
reply = new NetMQMessage ();
reply.Push (TitanicReturnCode.Ok.ToString ());
}
}
}
/// <summary>
/// adds an empty frame and a specified frame to a message
/// </summary>
private static NetMQMessage Wrap (NetMQFrame frame, NetMQMessage message)
{
var result = new NetMQMessage (message);
if (frame.BufferSize > 0)
{
result.Push (NetMQFrame.Empty);
result.Push (frame);
}
return result;
}
/// <summary>
/// process any titanic reply request by a client
///
/// <para>will send an OK, PENDING or UNKNOWN as result of the request for the reply</para>
/// </summary>
internal void ProcessTitanicReply( IMDPWorker mdpWorker = null)
{
// get a MDP worker with an automatic id and register with the service "titanic.reply"
// the worker will automatically start and connect to the indicated address
using (var worker = mdpWorker ?? new MDPWorker (m_titanicAddress, TitanicOperation.Reply.ToString ()))
{
NetMQMessage reply = null;
while (true)
{
// initiate the communication to MDP Broker with sending a 'null',
// since there is no initial reply everytime thereafter the reply will be send
var request = worker.Receive (reply);
Log (string.Format ("TITANIC REPLY] received: {0}", request));
//! has there been a breaking cause? -> exit
if (ReferenceEquals (request, null))
break;
var requestIdAsString = request.Pop ().ConvertToString ();
var requestId = Guid.Parse (requestIdAsString);
if (m_io.ExistsMessage (TitanicOperation.Reply, requestId))
{
Log (string.Format ("[TITANIC REPLY] reply for request exists: {0}", requestId));
reply = m_io.GetMessage (TitanicOperation.Reply, requestId); // [service][reply]
reply.Push (TitanicReturnCode.Ok.ToString ()); // ["OK"][service][reply]
}
else
{
reply = new NetMQMessage ();
var replyCommand = (m_io.ExistsMessage (TitanicOperation.Request, requestId)
? TitanicReturnCode.Pending
: TitanicReturnCode.Unknown);
reply.Push (replyCommand.ToString ());
}
Log (string.Format ("[TITANIC REPLY] reply: {0}", reply));
}
}
}
public void Run()
{
Console.WriteLine("[CLIENT {0}] Starting", m_id);
var rnd = new Random(m_id);
// each request shall have an unique id in order to recognize an reply for a request
var messageId = new byte[5];
// create clientId for messages
var clientId = new[] { m_id };
// a flag to signal that an answer has arrived
bool messageAnswered = false;
// we use a poller because we have a socket and a timer to monitor
using (var clientPoller = new NetMQPoller())
using (var client = new RequestSocket())
using (var monitor = new PushSocket())
{
client.Connect(m_localFrontendAddress);
monitor.Connect(m_monitorAddress);
client.Options.Identity = new[] { m_id };
var timer = new NetMQTimer((int)TimeSpan.FromSeconds(10).TotalMilliseconds);
// use as flag to indicate exit
var exit = false;
// every 10 s check if message has been received, if not then send error message and ext
// and restart timer otherwise
timer.Elapsed += (s, e) =>
{
if (messageAnswered)
{
e.Timer.Enable = true;
}
else
{
var msg = string.Format("[CLIENT {0}] ERR - EXIT - lost message {1}", m_id, messageId);
// send an error message
monitor.SendFrame(msg);
// if poller is started than stop it
if (clientPoller.IsRunning)
{
clientPoller.Stop();
}
// mark the required exit
exit = true;
}
};
// process arriving answers
client.ReceiveReady += (s, e) =>
{
// mark the arrival of an answer
messageAnswered = true;
// worker is supposed to answer with our request id
var reply = e.Socket.ReceiveMultipartMessage();
if (reply.FrameCount == 0)
{
// something went wrong
monitor.SendFrame(string.Format("[CLIENT {0}] Received an empty message!", m_id));
// mark the exit flag to ensure the exit
exit = true;
}
else
{
var sb = new StringBuilder();
// create success message
foreach (var frame in reply)
{
sb.Append("[" + frame.ConvertToString() + "]");
}
// send the success message
monitor.SendFrame(string.Format("[CLIENT {0}] Received answer {1}",
m_id,
sb.ToString()));
}
};
// add socket & timer to poller
clientPoller.Add(client);
clientPoller.Add(timer);
// start poller in another thread to allow the continued processing
clientPoller.RunAsync();
// if true the message has been answered
// the 0th message is always answered
messageAnswered = true;
while (!exit)
{
// simulate sporadic activity by randomly delaying
Thread.Sleep((int)TimeSpan.FromSeconds(rnd.Next(5)).TotalMilliseconds);
// only send next message if the previous one has been replied to
if (messageAnswered)
{
// generate random 5 byte as identity for for the message
rnd.NextBytes(messageId);
messageAnswered = false;
// create message [client adr][empty][message id] and send it
var msg = new NetMQMessage();
msg.Push(messageId);
msg.Push(NetMQFrame.Empty);
msg.Push(clientId);
client.SendMultipartMessage(msg);
}
}
// stop poller if needed
if (clientPoller.IsRunning)
{
clientPoller.Stop();
}
}
}
/// <summary>
/// process a titanic request according to TITANIC Protocol
///
/// <para>it connects via provided PAIR socket to main thread</para>
/// <para>write request to disk and return the GUID to client</para>
/// sends the GUID of the request back via the pipe for further processing
/// </summary>
internal void ProcessTitanicRequest( PairSocket pipe, IMDPWorker mdpWorker = null)
{
// get a MDP worker with an automatic id and register with the service "titanic.request"
// the worker will automatically start and connect to a MDP Broker at the indicated address
using (var worker = mdpWorker ?? new MDPWorker (m_titanicAddress, TitanicOperation.Request.ToString ()))
{
NetMQMessage reply = null;
while (true)
{
// initiate the communication with sending a 'null', since there is no initial reply
// a request should be [service name][request data]
var request = worker.Receive (reply);
Log (string.Format ("[TITANIC REQUEST] Received request: {0}", request));
//! has there been a breaking cause? -> exit
if (ReferenceEquals (request, null))
break;
//! check if service exists! and return 'Unknown' if not
// generate Guid for the request
var requestId = Guid.NewGuid ();
// save request to file -> [service name][request data]
m_io.SaveMessage (TitanicOperation.Request, requestId, request);
Log (string.Format ("[TITANIC REQUEST] sending through pipe: {0}", requestId));
// send GUID through message queue to main thread
pipe.SendFrame (requestId.ToString ());
// return GUID via reply message via worker.Receive call
reply = new NetMQMessage ();
// [Guid]
reply.Push (requestId.ToString ());
// [Ok][Guid]
reply.Push (TitanicReturnCode.Ok.ToString ());
Log (string.Format ("[TITANIC REQUEST] sending reply: {0}", reply));
}
}
}
/// <summary>
/// usage: MDPClientAsyncExample [-v] [-rn] (1 ;lt n ;lt 100000 / Default == 10)
///
/// implements a MDPClientAsync API usage
/// </summary>
private static void Main(string[] args)
{
if (args.Length == 1 || args.Length > 2)
{
if (!(args[0] == "-v" || args[0].Contains("-r")))
{
Console.WriteLine("MDPClientExample [-v(erbose) OR -h(elp)]");
Console.WriteLine("\t-v => verbose");
Console.WriteLine("\tto stop processing use CTRL+C");
return;
}
}
const string service_name = "echo";
const int max_runs = 100000;
bool verbose = false;
int runs = 10;
if (args.Length >= 1)
{
if (args.Length == 1 && args[0] == "-v")
{
verbose = true;
}
else if (args.Length == 2)
{
verbose = args[0] == "-v" || args[1] == "-v";
}
if (args[0].Contains("-r") || args.Length > 1)
{
if (args[0].Contains("-r"))
{
runs = GetInt(args[0]);
}
else if (args[1].Contains("-r"))
{
runs = GetInt(args[1]);
}
}
}
runs = runs == -1 ? 10 : runs > max_runs ? max_runs : runs;
var id = new[] { (byte)'C', (byte)'1' };
var watch = new Stopwatch();
Console.WriteLine("Starting MDPClient and will send {0} requests to service <{1}>.", runs, service_name);
Console.WriteLine("(writes '.' for every 100 and '|' for every 1000 requests)\n");
try
{
// create MDP client and set verboseness && use automatic disposal
using (var session = new MDPClientAsync("tcp://localhost:5555", id))
{
if (verbose)
{
session.LogInfoReady += (s, e) => Console.WriteLine("{0}", e.Info);
}
session.ReplyReady += (s, e) => Console.WriteLine("{0}", e.Reply);
// just give everything time to settle in
Thread.Sleep(500);
watch.Start();
for (int count = 0; count < runs; count++)
{
var request = new NetMQMessage();
// set the request data
request.Push("Hello World!");
// send the request to the service
session.Send(service_name, request);
if (count % 1000 == 0)
{
Console.Write("|");
}
else
if (count % 100 == 0)
{
Console.Write(".");
}
}
watch.Stop();
Console.Write("\nStop receiving with any key!");
Console.ReadKey();
}
}
catch (Exception ex)
{
Console.WriteLine("ERROR:");
Console.WriteLine(ex.Message);
Console.WriteLine(ex.StackTrace);
return;
}
var time = watch.Elapsed;
Console.WriteLine("{0} request/replies in {1} ms processed! Took {2:N3} ms per REQ/REP",
runs,
time.TotalMilliseconds,
time.TotalMilliseconds / runs);
Console.Write("\nExit with any key!");
Console.ReadKey();
}
/// <summary>
/// carry out the actual call to the worker via a broker in order
/// to process the request and get the reply which we wait for
///
/// <para>it creates a MDPClient and requests from MDPBroker the service.
/// if that service exists it uses the returned worker address and issues
/// a request with the appropriate data and waits for the reply</para>
/// </summary>
/// <param name="serviceName">the service's name requested</param>
/// <param name="request">request to process by worker</param>
/// <param name="serviceClient">mdp client handling the forwarding of requests
/// to service offering mdp worker via mdp broker and collecting
/// the replies from the mdp worker</param>
/// <returns><c>true</c> if successfull and <c>false</c> otherwise</returns>
private NetMQMessage ServiceCall( string serviceName, NetMQMessage request, IMDPClient serviceClient = null)
{
// create MDPClient session and send the request to MDPBroker
using (var session = serviceClient ?? new MDPClient (m_titanicAddress))
{
session.Timeout = TimeSpan.FromMilliseconds (1000); // 1s
session.Retries = 1; // only 1 retry
// use MMI protocol to check if service is available
var mmi = new NetMQMessage ();
// add name of service to inquire
mmi.Push (serviceName);
// request mmi.service resolution
var reply = session.Send ("mmi.service", mmi);
// first frame should be result of inquiry
var rc = reply[0].ConvertToString ();
var answer = (MmiCode) Enum.Parse (typeof (MmiCode), rc);
// answer == "Ok" -> service is available -> make the request
if (answer == MmiCode.Ok)
{
Log (string.Format ("[TITANIC SERVICECALL] -> {0} - {1}", serviceName, request));
return session.Send (serviceName, request);
}
Log (string.Format ("[TITANIC SERVICECALL] Service {0} (RC = {1}/{2}) is not available.",
serviceName,
answer,
request));
//! shall this information be available for request? Unknown or Pending
//! so TitanicRequest could utilize this information
return null;
}
}
