public static void Main(string[] args)
{
using (var context = new Context(1))
{
// Connect to task ventilator and weather server
using (Socket receiver = context.Socket(SocketType.PULL), subscriber = context.Socket(SocketType.SUB))
{
receiver.Connect("tcp://localhost:5557");
subscriber.Connect("tcp://localhost:5556");
subscriber.Subscribe("10001 ", Encoding.Unicode);
var items = new PollItem[2];
items[0] = receiver.CreatePollItem(IOMultiPlex.POLLIN);
items[0].PollInHandler += ReceiverPollInHandler;
items[1] = subscriber.CreatePollItem(IOMultiPlex.POLLIN);
items[1].PollInHandler += SubscriberPollInHandler;
// Process messages from both sockets
while (true)
{
context.Poll(items, -1);
}
}
}
}
static void Main(string[] args)
{
using (var context = new Context(1))
using (var receiver = context.Socket(SocketType.PULL))
{
receiver.Connect("tcp://localhost:5557");
using (var sender = context.Socket(SocketType.PUSH))
{
sender.Connect("tcp://localhost:5558");
using (var controller = context.Socket(SocketType.SUB))
{
controller.Connect("tcp://localhost:5559");
controller.Subscribe(string.Empty, Encoding.Unicode);
bool run = true;
PollItem[] items = new PollItem[2];
items[0] = receiver.CreatePollItem(IOMultiPlex.POLLIN);
items[0].PollInHandler += (socket, revents) => ReceiverPollInHandler(socket, sender);
items[1] = controller.CreatePollItem(IOMultiPlex.POLLIN);
items[1].PollInHandler += delegate { run = false; };
// Process tasks as long as the controller does not signal the end.
while (run)
{
context.Poll(items);
}
}
}
}
}
public void Broker()
{
// Initialize poll set
PollItem[] pollItems = new PollItem[2];
pollItems[0] = frontend.CreatePollItem(IOMultiPlex.POLLIN);
pollItems[0].PollInHandler += new PollHandler(FrontendPollInHandler);
pollItems[1] = backend.CreatePollItem(IOMultiPlex.POLLIN);
pollItems[1].PollInHandler += new PollHandler(BackendPollInHandler);
// Switch messages between sockets
while (true) {
context.Poll(pollItems, -1);
}
}
public void Run()
{
using (var context = new Context(1))
{
using (var pub = context.Socket(SocketType.PUB))
using (var control = context.Socket(SocketType.SUB))
{
pub.Bind("inproc://workers");
var step2 = new Thread(() => Worker(context));
step2.Start();
Thread.Sleep(100);
control.Connect("inproc://control");
control.Subscribe("", Encoding.Unicode);
var count = 0;
var message = string.Empty;
var controlItems = new PollItem[2];
controlItems[0] = control.CreatePollItem(IOMultiPlex.POLLIN);
controlItems[0].PollInHandler += (x, y) =>
{
message = x.Recv(Encoding.Unicode);
Console.WriteLine("A message: {0}", message);
};
controlItems[1] = pub.CreatePollItem(IOMultiPlex.POLLOUT);
controlItems[1].PollOutHandler += (z, y) =>
{
Console.WriteLine(string.Format("About to send {0}", count));
z.Send(count++.ToString(), Encoding.Unicode);
};
//Console.WriteLine("Recieved the {0} signal",control.Recv(Encoding.Unicode));
while (string.IsNullOrEmpty(message) || !message.Equals("Stop"))
{
context.Poll(controlItems, -1);
Thread.Sleep(500);
}
}
}
}
public static void Main(string[] args)
{
using (var context = new Context(1))
{
using (Socket frontend = context.Socket(SocketType.ROUTER), backend = context.Socket(SocketType.DEALER))
{
frontend.Bind("tcp://*:5559");
backend.Bind("tcp://*:5560");
var pollItems = new PollItem[2];
pollItems[0] = frontend.CreatePollItem(IOMultiPlex.POLLIN);
pollItems[0].PollInHandler += (socket, revents) => FrontendPollInHandler(socket, backend);
pollItems[1] = backend.CreatePollItem(IOMultiPlex.POLLIN);
pollItems[1].PollInHandler += (socket, revents) => BackendPollInHandler(socket, frontend);
while (true)
{
context.Poll(pollItems, -1);
}
}
}
}
public void Worker(object things)
{
{
var context = (Context)things;
using (var control = context.Socket(SocketType.PUB))
using (var sub = context.Socket(SocketType.SUB))
{
Console.WriteLine("Worker started");
control.Bind("inproc://control");
sub.Subscribe("", Encoding.Unicode);
sub.Connect("inproc://workers");
var count = 0;
var items = new PollItem[1];
items[0] = sub.CreatePollItem(IOMultiPlex.POLLIN);
items[0].PollInHandler += (x, y) =>
{
Console.WriteLine("Received : {0}",x.Recv(Encoding.Unicode));
count++;
};
control.Send("Go", Encoding.Unicode);
while (count < 10)
{
context.Poll(items, -1);
}
control.Send("Stop", Encoding.Unicode);
}
}
}
private void CommonInit(bool local)
{
_msg = Marshal.AllocHGlobal(ZMQ_MSG_T_SIZE);
_localSocket = local;
_pollItem = new PollItem(new ZMQPollItem(Ptr, 0, 0), this);
}
/// <summary>
/// Polls the supplied items for events, with infinite timeout.
/// </summary>
/// <param name="items">Items to Poll</param>
/// <returns>Number of Poll items with events</returns>
public int Poll(PollItem[] items)
{
return Poll(items, -1);
}
/// <summary>
/// Polls the supplied items for events.
/// </summary>
/// <param name="items">Items to Poll</param>
/// <param name="timeout">Timeout(micro seconds)</param>
/// <returns>Number of Poll items with events</returns>
public int Poll(PollItem[] items, long timeout)
{
return Poller(items, timeout);
}
/// <summary>
/// Polls the supplied items for events, with infinite timeout.
/// Static method
/// </summary>
/// <param name="items">Items to Poll</param>
/// <returns>Number of Poll items with events</returns>
public static int Poller(PollItem[] items)
{
return Poller(items, -1);
}
/// <summary>
/// Polls the supplied items for events. Static method.
/// </summary>
/// <param name="items">Items to Poll</param>
/// <param name="timeout">Timeout(micro seconds)</param>
/// <returns>Number of Poll items with events</returns>
public static int Poller(PollItem[] items, long timeout)
{
var spentTimeout = new Stopwatch();
int rc = -1;
if (timeout >= 0) {
spentTimeout.Start();
}
while (rc != 0) {
var zitems = new ZMQPollItem[items.Length];
int index = 0;
foreach (PollItem item in items) {
item.ZMQPollItem.ResetRevents();
zitems[index] = item.ZMQPollItem;
index++;
}
rc = C.zmq_poll(zitems, items.Length, timeout);
if (rc > 0) {
for (index = 0; index < items.Length; index++) {
items[index].ZMQPollItem = zitems[index];
items[index].FireEvents();
}
break;
}
if (rc < 0) {
if (C.zmq_errno() == 4) {
if (spentTimeout.IsRunning) {
long elapsed = spentTimeout.ElapsedMilliseconds * 1000;
if (timeout < elapsed) {
break;
}
timeout -= elapsed;
continue;
}
continue;
}
throw new Exception();
}
}
return rc;
}
public void Run(object threadContext)
{
object[] thrContextObjects = (object[])threadContext;
Context ctx = (Context)thrContextObjects[0];
using (Socket ctl = ctx.Socket(SocketType.REP))
{
// Bind the control reply socket:
ctl.SndBuf = 1048576 * 20;
if (host.tsp == Transport.TCP)
{
ctl.Bind(Transport.TCP, host.device, (host.port + 1));
}
else
{
ctl.Bind(Transport.TCP, "*", (host.port + 1));
}
// Wait for the sockets to bind:
Thread.Sleep(500);
// Create a poller on the control socket to handle client requests:
PollItem[] pollItems = new PollItem[1];
pollItems[0] = ctl.CreatePollItem(IOMultiPlex.POLLIN);
pollItems[0].PollInHandler += new PollHandler((Socket sock, IOMultiPlex mp) =>
{
Queue<byte[]> request = sock.RecvAll();
// Bad message?
if (request.Count == 0) return;
Guid clientIdentity = new Guid(request.Dequeue().Skip(1).ToArray());
ClientState client;
bool newClient = false;
lock (host.clientLock)
{
if (!host.clients.ContainsKey(clientIdentity))
{
client = new ClientState(clientIdentity);
host.clients.Add(clientIdentity, client);
newClient = true;
}
else
{
client = host.clients[clientIdentity];
}
// Update last message received time to now:
client.LastMessageTime = DateTimeOffset.UtcNow;
}
string cmd = Encoding.Unicode.GetString(request.Dequeue());
// Process the client command:
trace("{0}: Client sent '{1}'", clientIdentity.ToString(), cmd);
switch (cmd)
{
case "JOIN":
sock.SendMore("JOINED", Encoding.Unicode);
ReadOnlyCollection<FileInfo> files = host.tarball.Files;
sock.SendMore(BitConverter.GetBytes(host.numChunks));
sock.SendMore(BitConverter.GetBytes(host.chunkSize));
sock.SendMore(BitConverter.GetBytes(host.tarball.Files.Count));
foreach (var fi in files)
{
string fiName = fi.FullName.Substring(host.basePath.Length);
sock.SendMore(fiName, Encoding.Unicode);
sock.SendMore(BitConverter.GetBytes(fi.Length));
sock.SendMore(new byte[16]);
}
sock.Send("", Encoding.Unicode);
trace("{0}: Sent JOINED response", clientIdentity.ToString());
if (host.ClientJoined != null) host.ClientJoined(host, clientIdentity);
break;
case "NAKS":
// Receive the client's current NAK:
byte[] tmp = request.Dequeue();
if (tmp.Length != host.numBitArrayBytes)
{
trace("{0}: Bad NAKs", clientIdentity.ToString());
sock.Send("BADNAKS", Encoding.Unicode);
break;
}
sock.Send("", Encoding.Unicode);
lock (host.clientLock)
{
BitArray newNAKs = new BitArray(tmp);
// Add to the running ACK count the number of chunks turned from NAK to ACK in this update:
if (client.HasNAKs)
{
client.RunningACKCount += (
from i in Enumerable.Range(0, host.numChunks)
where (client.NAK[i] == true) && (newNAKs[i] == false)
select i
).Count();
}
else
{
client.RunningACKCount += newNAKs.Cast<bool>().Take(host.numChunks).Count(b => !b);
}
// Update to the new NAK state:
client.NAK = newNAKs;
// Inform the server to update the program:
host.haveNewNAKs = true;
}
trace("{0}: NAKs received", clientIdentity.ToString());
if (host.ChunksACKed != null) host.ChunksACKed(host);
break;
case "LEAVE":
sock.Send("LEFT", Encoding.Unicode);
lock (host.clientLock)
{
// Remove the client's state record:
host.clients.Remove(clientIdentity);
}
if (host.ClientLeft != null) host.ClientLeft(host, clientIdentity, ClientLeaveReason.Left);
trace("{0}: Client left", clientIdentity.ToString());
break;
case "ALIVE":
if (newClient)
{
trace("{0}: WHOAREYOU", clientIdentity.ToString());
byte[] sendIdentity = new byte[1] { (byte)'@' }.Concat(clientIdentity.ToByteArray()).ToArray();
sock.Send("WHOAREYOU", Encoding.Unicode);
break;
}
// Fantastic.
sock.Send("", Encoding.Unicode);
break;
default:
// Unknown command.
sock.Send("UNKNOWN", Encoding.Unicode);
trace("{0}: Unknown request", clientIdentity.ToString());
break;
}
request = null;
});
while (host.isRunning)
{
while (ctx.Poll(pollItems, 100000L) == 1)
{
}
}
}
}
private void DiskWriterThread(object threadContext)
{
try
{
Context ctx = (Context)threadContext;
using (Socket
disk = new Socket(SocketType.PULL),
diskACK = new Socket(SocketType.PUSH)
)
{
if (diskHWM > 0) disk.RCVHWM = diskHWM;
//disk.RcvBuf = 1048576 * diskHWM * 4;
disk.Connect("inproc://disk");
diskACK.Connect("inproc://diskack");
// 'inproc://' connections are immediate. No need to sleep.
//Thread.Sleep(500);
bool done = false;
PollItem[] pollItems = new PollItem[1];
pollItems[0] = disk.CreatePollItem(IOMultiPlex.POLLIN);
pollItems[0].PollInHandler += new PollHandler((Socket sock, IOMultiPlex revents) =>
{
Queue<byte[]> packet = sock.RecvAll();
// Determine the action to take:
byte[] cmd = packet.Dequeue();
int chunkIdx;
byte[] chunkIdxPkt;
byte[] chunk;
switch (cmd[0])
{
// Write to disk
case (byte)'W':
chunkIdx = BitConverter.ToInt32(chunkIdxPkt = packet.Dequeue(), 0);
chunk = packet.Dequeue();
if (!testMode)
{
lock (tarballLock)
{
tarball.Seek((long)chunkIdx * chunkSize, SeekOrigin.Begin);
tarball.Write(chunk, 0, chunk.Length);
}
}
// Notify the host that a chunk was written:
if (ChunkWritten != null) ChunkWritten(this, chunkIdx);
// Send the acknowledgement that this chunk was written to disk:
diskACK.Send(chunkIdxPkt);
break;
// Exit thread
case (byte)'X':
done = true;
break;
default:
break;
}
});
while (!done)
{
if (ctx.Poll(pollItems, 1000L) == 0)
{
Thread.Sleep(5);
}
}
}
}
catch (System.Exception ex)
{
Console.Error.WriteLine(ex.ToString());
}
}
/// <summary>
/// Main thread to host the client process.
/// </summary>
/// <param name="threadContext">A ZMQ.Context instance.</param>
public void Run(object threadContext)
{
try
{
Context ctx = (Context)threadContext;
// This would be a using statement on data and ctl but we need to close and reopen sockets, which means
// we have to Dispose() of a socket early and create a new instance. C#, wisely so, prevents you from
// reassigning a using variable.
Socket data = null, ctl = null, disk = null, diskACK = null;
try
{
data = ctx.Socket(SocketType.SUB);
ctl = ctx.Socket(SocketType.REQ);
disk = new Socket(SocketType.PUSH);
diskACK = new Socket(SocketType.PULL);
// Set the HWM for the disk PUSH so that the PUSH blocks if the PULL can't keep up writing:
if (diskHWM > 0) disk.SNDHWM = diskHWM;
//disk.SndBuf = 1048576 * diskHWM * 4;
// inproc:// binds are immediate.
disk.Bind("inproc://disk");
diskACK.Bind("inproc://diskack");
// Create the disk PULL thread:
Thread diskWriterThread = new Thread(new ParameterizedThreadStart(DiskWriterThread));
diskWriterThread.Start(ctx);
data.RCVHWM = networkHWM;
//data.RcvBuf = 1048576 * hwm * 4;
// NOTE: work-around for MS bug in WinXP's networking stack. See http://support.microsoft.com/kb/201213 for details.
data.RcvBuf = 0;
if (tsp == Transport.EPGM || tsp == Transport.PGM)
{
data.Rate = pgmRate;
}
string address = tsp.ToString().ToLower() + "://" + deviceData + ":" + portData;
Console.WriteLine("Connect(\"{0}\")", address);
data.Connect(address);
data.Subscribe(subscription, Encoding.Unicode);
// Connect to the control request socket:
ctl.RcvBuf = 1048576 * 4;
ctl.SndBuf = 1048576 * 4;
address = "tcp://" + deviceCtl + ":" + portCtl.ToString();
Console.WriteLine("Connect(\"{0}\")", address);
ctl.Connect(address);
Guid myIdentity = Guid.NewGuid();
ctl.Identity = new byte[1] { (byte)'@' }.Concat(myIdentity.ToByteArray()).ToArray();
Thread.Sleep(500);
// Begin client logic:
naks = null;
numBytes = 0;
int ackCount = -1;
byte[] nakBuf = null;
bool shuttingDown = false;
Stopwatch recvTimer = Stopwatch.StartNew();
long lastElapsedMilliseconds = recvTimer.ElapsedMilliseconds;
int msgsRecv = 0, msgsWritten = 0;
try
{
// Poll for incoming messages on the data SUB socket:
PollItem[] pollItems = new PollItem[3];
pollItems[0] = data.CreatePollItem(IOMultiPlex.POLLIN);
pollItems[1] = ctl.CreatePollItem(IOMultiPlex.POLLIN /* | IOMultiPlex.POLLOUT */);
pollItems[2] = diskACK.CreatePollItem(IOMultiPlex.POLLIN);
ZMQStateMasheen<DataSUBState> dataFSM;
ZMQStateMasheen<ControlREQState> controlFSM;
Queue<QueuedControlMessage> controlStateQueue = new Queue<QueuedControlMessage>();
DateTimeOffset lastSentNAKs = DateTimeOffset.UtcNow.Subtract(TimeSpan.FromSeconds(60));
// We create a state machine to handle our send/recv state:
dataFSM = new ZMQStateMasheen<DataSUBState>(
// Set the initial state:
DataSUBState.Recv,
// Initial state handler:
new ZMQStateMasheen<DataSUBState>.State(DataSUBState.Recv, (sock, revents) =>
{
Queue<byte[]> packet = sock.RecvAll();
if (packet.Count == 0) { trace("FAIL!"); return DataSUBState.Recv; }
string sub = Encoding.Unicode.GetString(packet.Dequeue());
if (packet.Count == 0) { trace("FAIL!"); return DataSUBState.Recv; }
string cmd = Encoding.Unicode.GetString(packet.Dequeue());
if ((naks != null) && (cmd == "DATA"))
{
if (packet.Count == 0) { trace("FAIL!"); return DataSUBState.Recv; }
int chunkIdx = BitConverter.ToInt32(packet.Dequeue(), 0);
// Count the number of messages received from the network:
++msgsRecv;
if (ChunkReceived != null) ChunkReceived(this, chunkIdx);
// Already received this chunk?
if (!naks[chunkIdx])
{
trace("ALREADY RECV {0}", chunkIdx);
return DataSUBState.Recv;
}
trace("RECV {0}", chunkIdx);
// Queue up the disk writes with PUSH/PULL and an HWM on a separate thread to maintain as
// constant disk write throughput as we can get... The HWM will enforce the PUSHer to block
// when the PULLer cannot receive yet.
if (packet.Count == 0) return DataSUBState.Recv;
byte[] chunk = packet.Dequeue();
disk.SendMore(cmdWrite);
disk.SendMore(BitConverter.GetBytes(chunkIdx));
// This will block if the disk queue gets full:
disk.Send(chunk);
chunk = null;
return DataSUBState.Recv;
}
else if (cmd == "PING")
{
if (shuttingDown) return DataSUBState.Recv;
controlStateQueue.Enqueue(new QueuedControlMessage(ControlREQState.SendALIVE, null));
return DataSUBState.Recv;
}
else
{
return DataSUBState.Recv;
}
})
);
// We create a state machine to handle our send/recv state:
object tmpControlState = null;
controlFSM = new ZMQStateMasheen<ControlREQState>(
ControlREQState.Nothing,
new ZMQStateMasheen<ControlREQState>.State(ControlREQState.Nothing, (sock, revents) =>
{
// Do nothing state.
return ControlREQState.Nothing;
}),
new ZMQStateMasheen<ControlREQState>.State(ControlREQState.SendALIVE, (sock, revents) =>
{
if (shuttingDown) return ControlREQState.Nothing;
ctl.SendMore(ctl.Identity);
ctl.Send("ALIVE", Encoding.Unicode);
return new ZMQStateMasheen<ControlREQState>.MoveOperation(ControlREQState.RecvALIVE, false);
}),
new ZMQStateMasheen<ControlREQState>.State(ControlREQState.RecvALIVE, (sock, revents) =>
{
Queue<byte[]> packet = ctl.RecvAll();
if (packet.Count == 0) return ControlREQState.Nothing;
string cmd = Encoding.Unicode.GetString(packet.Dequeue());
//trace("Server: '{0}'", cmd);
if (cmd == "") return ControlREQState.Nothing;
else if (cmd == "WHOAREYOU") return new ZMQStateMasheen<ControlREQState>.MoveOperation(ControlREQState.SendJOIN, true);
return ControlREQState.Nothing;
}),
new ZMQStateMasheen<ControlREQState>.State(ControlREQState.SendJOIN, (sock, revents) =>
{
if (shuttingDown) return ControlREQState.Nothing;
// Send a JOIN request:
ctl.SendMore(ctl.Identity);
ctl.Send("JOIN", Encoding.Unicode);
return ControlREQState.RecvJOIN;
}),
new ZMQStateMasheen<ControlREQState>.State(ControlREQState.RecvJOIN, (sock, revents) =>
{
Queue<byte[]> packet = ctl.RecvAll();
if (packet.Count == 0) return ControlREQState.Nothing;
string resp = Encoding.Unicode.GetString(packet.Dequeue());
if (resp != "JOINED")
{
// TODO: handle this failure.
trace("FAIL!"); return ControlREQState.Nothing;
}
// TODO: make this atomically succeed or fail without corrupting state
if (packet.Count == 0) { trace("FAIL!"); return ControlREQState.Nothing; }
numChunks = BitConverter.ToInt32(packet.Dequeue(), 0);
numBytes = ((numChunks + 7) & ~7) >> 3;
nakBuf = new byte[numBytes];
if (packet.Count == 0) { trace("FAIL!"); return ControlREQState.Nothing; }
chunkSize = BitConverter.ToInt32(packet.Dequeue(), 0);
if (packet.Count == 0) { trace("FAIL!"); return ControlREQState.Nothing; }
int numFiles = BitConverter.ToInt32(packet.Dequeue(), 0);
List<TarballEntry> tbes = new List<TarballEntry>(numFiles);
for (int i = 0; i < numFiles; ++i)
{
if (packet.Count == 0) { trace("FAIL!"); return ControlREQState.Nothing; }
string fiName = Encoding.Unicode.GetString(packet.Dequeue());
if (packet.Count == 0) { trace("FAIL!"); return ControlREQState.Nothing; }
long length = BitConverter.ToInt64(packet.Dequeue(), 0);
if (packet.Count == 0) { trace("FAIL!"); return ControlREQState.Nothing; }
byte[] hash = packet.Dequeue();
TarballEntry tbe = new TarballEntry(fiName, length, hash);
tbes.Add(tbe);
}
lock (tarballLock)
{
if (tarball != null)
{
tarball.Close();
tarball.Dispose();
tarball = null;
}
}
lock (tarballLock)
{
// Create the tarball reader that writes the files locally:
tarball = new TarballStreamReader(downloadDirectory, tbes);
// Get our local download state:
naks = getClientState(this, tarball);
}
ackCount = naks.Cast<bool>().Take(numChunks).Count(b => !b);
return new ZMQStateMasheen<ControlREQState>.MoveOperation(ControlREQState.SendNAKS, true);
}),
new ZMQStateMasheen<ControlREQState>.State(ControlREQState.SendNAKS, (sock, revents) =>
{
if (shuttingDown) return ControlREQState.Nothing;
if (nakBuf == null) return ControlREQState.Nothing;
// Send our NAKs:
ctl.SendMore(ctl.Identity);
ctl.SendMore("NAKS", Encoding.Unicode);
// TODO: RLE!
naks.CopyTo(nakBuf, 0);
trace("SEND NAK");
ctl.Send(nakBuf);
return ControlREQState.RecvNAKS;
}),
new ZMQStateMasheen<ControlREQState>.State(ControlREQState.RecvNAKS, (sock, revents) =>
{
Queue<byte[]> packet = ctl.RecvAll();
// Don't care what the response is for now.
return ControlREQState.Nothing;
})
);
// Disk-write acknowledgement poll input handler:
pollItems[2].PollInHandler += new PollHandler((Socket sock, IOMultiPlex revents) =>
{
byte[] idxPkt = sock.RecvAll().Dequeue();
int chunkIdx = BitConverter.ToInt32(idxPkt, 0);
naks[chunkIdx] = false;
// Count the number of messages written to disk:
++msgsWritten;
++ackCount;
// Profiled - terrible performance. Not a big surprise here.
//ackCount = naks.Cast<bool>().Take(numChunks).Count(b => !b);
// If we ran up the timer, send more ACKs:
if (DateTimeOffset.UtcNow.Subtract(lastSentNAKs).TotalMilliseconds >= 250d)
{
lastSentNAKs = DateTimeOffset.UtcNow;
controlStateQueue.Enqueue(new QueuedControlMessage(ControlREQState.SendNAKS, null));
}
});
pollItems[0].PollInHandler += new PollHandler(dataFSM.StateMasheen);
pollItems[1].PollInHandler += new PollHandler(controlFSM.StateMasheen);
DateTimeOffset lastRecv = DateTimeOffset.UtcNow;
// Start the control state machine with a JOIN:
controlStateQueue.Enqueue(new QueuedControlMessage(ControlREQState.SendJOIN, null));
// Create a socket poller for the data socket:
while (true)
{
// If we disposed of the previous control socket, create a new one:
if (ctl == null)
{
trace("Creating new CONTROL socket");
// Set up new socket:
ctl = ctx.Socket(SocketType.REQ);
// Connect to the control request socket:
ctl.Connect("tcp://" + deviceCtl + ":" + portCtl.ToString());
ctl.Identity = new byte[1] { (byte)'@' }.Concat(myIdentity.ToByteArray()).ToArray();
}
if (!shuttingDown && IsSendState(controlFSM.CurrentState))
{
// If we ran up the timer, send more NAKs:
if (DateTimeOffset.UtcNow.Subtract(lastSentNAKs).TotalMilliseconds >= 200d)
{
lastSentNAKs = DateTimeOffset.UtcNow;
controlStateQueue.Enqueue(new QueuedControlMessage(ControlREQState.SendNAKS, null));
}
if (controlStateQueue.Count > 0)
{
var msg = controlStateQueue.Dequeue();
// Roll up similar state requests:
int dupes = 0;
while (controlStateQueue.Count > 0)
{
if (controlStateQueue.Peek().NewState != msg.NewState) break;
msg = controlStateQueue.Dequeue();
++dupes;
}
if (dupes > 0)
{
trace("Removed {0} redundant queued states", dupes);
}
tmpControlState = msg.Object;
// Run the control state machine to SEND:
controlFSM.CurrentState = msg.NewState;
controlFSM.StateMasheen(ctl, IOMultiPlex.POLLOUT);
}
}
// Measure our message send rate per minute:
long elapsed = recvTimer.ElapsedMilliseconds - lastElapsedMilliseconds;
if (elapsed >= 100L)
{
NetworkRecvChunksPerMinute = (int)((msgsRecv * 60000L) / elapsed);
DiskWriteChunksPerMinute = (int)((msgsWritten * 60000L) / elapsed);
lastElapsedMilliseconds = recvTimer.ElapsedMilliseconds;
msgsRecv = 0;
msgsWritten = 0;
}
if (!recvTimer.IsRunning)
{
recvTimer.Reset();
recvTimer.Start();
lastElapsedMilliseconds = 0L;
}
if (ackCount >= numChunks)
{
disk.Send(cmdExit);
break;
}
if (ctx.Poll(pollItems, 100L) == 0)
Thread.Sleep(1);
}
// Wait for the disk writer to finish up:
shuttingDown = true;
diskWriterThread.Join();
// If we were last to await some response, receive it:
if (controlFSM.CurrentState == ControlREQState.RecvALIVE || controlFSM.CurrentState == ControlREQState.RecvJOIN || controlFSM.CurrentState == ControlREQState.RecvNAKS)
{
PollItem[] recvPoll = new PollItem[1];
recvPoll[0] = ctl.CreatePollItem(IOMultiPlex.POLLIN);
recvPoll[0].PollInHandler += new PollHandler(controlFSM.StateMasheen);
// Wait a bit to receive the response:
for (int i = 0; (i < 20) && ((ctx.Poll(recvPoll, 100) == 1) && (ctl != null)); ++i)
{
}
}
// Send the LEAVE message:
ctl.SendMore(ctl.Identity);
ctl.Send("LEAVE", Encoding.Unicode);
Completed = true;
}
finally
{
if (tarball != null) tarball.Dispose();
}
}
finally
{
if (data != null) data.Dispose();
if (ctl != null) ctl.Dispose();
}
}
catch (System.Exception ex)
{
Console.Error.WriteLine(ex.ToString());
}
}
private void CommonInit(bool local)
{
_msg = Marshal.AllocHGlobal(ZMQ_MSG_T_SIZE);
_localSocket = local;
_pollItem = new PollItem(new ZMQPollItem(_ptr, 0, 0), this);
}
public static void Main(string[] args)
{
if (args.Length < 2)
{
Console.WriteLine("Usage: peering2 <myself> <peer_1> … <peer_N>");
return;
}
var myself = args[0];
Console.WriteLine("Hello, I am " + myself);
using (var context = new Context(1))
{
using (Socket cloudfe = context.Socket(SocketType.ROUTER), cloudbe = context.Socket(SocketType.ROUTER),
localfe = context.Socket(SocketType.ROUTER), localbe = context.Socket(SocketType.ROUTER))
{
cloudFeAddress = "tcp://127.0.0.1:" + myself;
cloudfe.Identity = Encoding.Unicode.GetBytes(myself);
cloudfe.Bind(cloudFeAddress);
cloudbe.Identity = Encoding.Unicode.GetBytes(myself);
for (int arg = 1; arg < args.Length; arg++)
{
var endpoint = "tcp://127.0.0.1:" + args[arg];
peers.Add(endpoint);
Console.WriteLine("I: connecting to cloud frontend at " + endpoint);
cloudbe.Connect(endpoint);
}
localFeAddress = cloudFeAddress + "1";
localfe.Bind(localFeAddress);
localBeAddress = cloudFeAddress + "2";
localbe.Bind(localBeAddress);
Console.WriteLine("Press Enter when all brokers are started: ");
Console.ReadKey();
var workers = new List<Thread>();
for (int workerNumber = 0; workerNumber < numberOfWorkers; workerNumber++)
{
workers.Add(new Thread(WorkerTask));
workers[workerNumber].Start();
}
var clients = new List<Thread>();
for (int clientNumber = 0; clientNumber < numberOfClients; clientNumber++)
{
clients.Add(new Thread(ClientTask));
clients[clientNumber].Start();
}
var workerQueue = new Queue<byte[]>();
var localfeReady = false;
var cloudfeReady = false;
var backends = new PollItem[2];
backends[0] = localbe.CreatePollItem(IOMultiPlex.POLLIN);
backends[0].PollInHandler += (socket, revents) =>
{
var zmsg = new ZMessage(socket);
// Use worker address for LRU routing
workerQueue.Enqueue(zmsg.Unwrap());
if (zmsg.BodyToString() != "READY")
{
SendReply(zmsg, cloudfe, localfe);
}
};
backends[1] = cloudbe.CreatePollItem(IOMultiPlex.POLLIN);
backends[1].PollInHandler += (socket, revents) =>
{
var zmsg = new ZMessage(socket);
// We don't use peer broker address for anything
zmsg.Unwrap();
SendReply(zmsg, cloudfe, localfe);
};
var frontends = new PollItem[2];
frontends[0] = cloudfe.CreatePollItem(IOMultiPlex.POLLIN);
frontends[0].PollInHandler += (socket, revents) =>
{
cloudfeReady = true;
};
frontends[1] = localfe.CreatePollItem(IOMultiPlex.POLLIN);
frontends[1].PollInHandler += (socket, revents) =>
{
localfeReady = true;
};
while (true)
{
var timeout = (workerQueue.Count > 0 ? 1000000 : -1);
var rc = Context.Poller(backends, timeout);
if (rc == -1)
break; // Interrupted
while (workerQueue.Count > 0)
{
Context.Poller(frontends, 0);
bool reRoutable;
ZMessage msg;
if (cloudfeReady)
{
cloudfeReady = false;
msg = new ZMessage(cloudfe);
reRoutable = false;
}
else if (localfeReady)
{
localfeReady = false;
msg = new ZMessage(localfe);
reRoutable = true;
}
else
{
break;
}
//if (reRoutable && workerQueue.Count > 0 && randomizer.Next(3) == 0)
if (reRoutable && peers.Count > 0 && randomizer.Next(4) == 0)
{
var randomPeer = randomizer.Next(1, args.Length - 1);
var endpoint = "tcp://127.0.0.1:" + args[randomPeer];
msg.Wrap(Encoding.Unicode.GetBytes(endpoint), new byte[0]);
msg.Send(cloudbe);
}
else
{
msg.Wrap(workerQueue.Dequeue(), new byte[0]);
msg.Send(localbe);
}
}
}
}
}
}
