public static bool[] GetBitArray(int x)
{
BitArray b = new BitArray(new int[] { x });
var bools = b.Cast<bool>();
//the visual representation is the opposite of how they bits actually go in the array so reverse things
return bools.Reverse().ToArray();
// bool[] bits = b.Cast<bool>().Select(bit => bit ? 1 : 0).ToArray();
}
internal static byte ConvertToByte(BitArray bits)
{
if (bits.Count != 8)
{
throw new ArgumentException("bits");
}
byte[] bytes = new byte[1];
var reversed = new BitArray(bits.Cast<bool>().Reverse().ToArray());
reversed.CopyTo(bytes, 0);
return bytes[0];
}
private static void PopulatePrimes(int upTo)
{
if(upTo <= s_Primes.DefaultIfEmpty().Max())
return;
var bitArray = new BitArray(upTo*100);
bitArray[0] = false;
bitArray[1] = false;
for (var i = 2; i < bitArray.Length; i++)
if (bitArray[i])
for (var j = i * 2; j < bitArray.Length; j += i)
bitArray[j] = false;
s_Primes = bitArray.Cast<bool>().Select((x, y) => new { x, y }).Where(x => x.x).Select(x => x.y).ToList();
}
public void GetLogicalDrives_Windows_MatchesExpectedLetters()
{
string[] drives = Environment.GetLogicalDrives();
uint mask = (uint)GetLogicalDrives();
var bits = new BitArray(new[] { (int)mask });
Assert.Equal(bits.Cast<bool>().Count(b => b), drives.Length);
for (int bit = 0, d = 0; bit < bits.Length; bit++)
{
if (bits[bit])
{
Assert.Contains((char)('A' + bit), drives[d++]);
}
}
}
public void Run(object threadContext)
{
object[] thrContextObjects = (object[])threadContext;
Context ctx = (Context)thrContextObjects[0];
using (Socket ctl = ctx.Socket(SocketType.REP))
{
// Bind the control reply socket:
ctl.SndBuf = 1048576 * 20;
if (host.tsp == Transport.TCP)
{
ctl.Bind(Transport.TCP, host.device, (host.port + 1));
}
else
{
ctl.Bind(Transport.TCP, "*", (host.port + 1));
}
// Wait for the sockets to bind:
Thread.Sleep(500);
// Create a poller on the control socket to handle client requests:
PollItem[] pollItems = new PollItem[1];
pollItems[0] = ctl.CreatePollItem(IOMultiPlex.POLLIN);
pollItems[0].PollInHandler += new PollHandler((Socket sock, IOMultiPlex mp) =>
{
Queue<byte[]> request = sock.RecvAll();
// Bad message?
if (request.Count == 0) return;
Guid clientIdentity = new Guid(request.Dequeue().Skip(1).ToArray());
ClientState client;
bool newClient = false;
lock (host.clientLock)
{
if (!host.clients.ContainsKey(clientIdentity))
{
client = new ClientState(clientIdentity);
host.clients.Add(clientIdentity, client);
newClient = true;
}
else
{
client = host.clients[clientIdentity];
}
// Update last message received time to now:
client.LastMessageTime = DateTimeOffset.UtcNow;
}
string cmd = Encoding.Unicode.GetString(request.Dequeue());
// Process the client command:
trace("{0}: Client sent '{1}'", clientIdentity.ToString(), cmd);
switch (cmd)
{
case "JOIN":
sock.SendMore("JOINED", Encoding.Unicode);
ReadOnlyCollection<FileInfo> files = host.tarball.Files;
sock.SendMore(BitConverter.GetBytes(host.numChunks));
sock.SendMore(BitConverter.GetBytes(host.chunkSize));
sock.SendMore(BitConverter.GetBytes(host.tarball.Files.Count));
foreach (var fi in files)
{
string fiName = fi.FullName.Substring(host.basePath.Length);
sock.SendMore(fiName, Encoding.Unicode);
sock.SendMore(BitConverter.GetBytes(fi.Length));
sock.SendMore(new byte[16]);
}
sock.Send("", Encoding.Unicode);
trace("{0}: Sent JOINED response", clientIdentity.ToString());
if (host.ClientJoined != null) host.ClientJoined(host, clientIdentity);
break;
case "NAKS":
// Receive the client's current NAK:
byte[] tmp = request.Dequeue();
if (tmp.Length != host.numBitArrayBytes)
{
trace("{0}: Bad NAKs", clientIdentity.ToString());
sock.Send("BADNAKS", Encoding.Unicode);
break;
}
sock.Send("", Encoding.Unicode);
lock (host.clientLock)
{
BitArray newNAKs = new BitArray(tmp);
// Add to the running ACK count the number of chunks turned from NAK to ACK in this update:
if (client.HasNAKs)
{
client.RunningACKCount += (
from i in Enumerable.Range(0, host.numChunks)
where (client.NAK[i] == true) && (newNAKs[i] == false)
select i
).Count();
}
else
{
client.RunningACKCount += newNAKs.Cast<bool>().Take(host.numChunks).Count(b => !b);
}
// Update to the new NAK state:
client.NAK = newNAKs;
// Inform the server to update the program:
host.haveNewNAKs = true;
}
trace("{0}: NAKs received", clientIdentity.ToString());
if (host.ChunksACKed != null) host.ChunksACKed(host);
break;
case "LEAVE":
sock.Send("LEFT", Encoding.Unicode);
lock (host.clientLock)
{
// Remove the client's state record:
host.clients.Remove(clientIdentity);
}
if (host.ClientLeft != null) host.ClientLeft(host, clientIdentity, ClientLeaveReason.Left);
trace("{0}: Client left", clientIdentity.ToString());
break;
case "ALIVE":
if (newClient)
{
trace("{0}: WHOAREYOU", clientIdentity.ToString());
byte[] sendIdentity = new byte[1] { (byte)'@' }.Concat(clientIdentity.ToByteArray()).ToArray();
sock.Send("WHOAREYOU", Encoding.Unicode);
break;
}
// Fantastic.
sock.Send("", Encoding.Unicode);
break;
default:
// Unknown command.
sock.Send("UNKNOWN", Encoding.Unicode);
trace("{0}: Unknown request", clientIdentity.ToString());
break;
}
request = null;
});
while (host.isRunning)
{
while (ctx.Poll(pollItems, 100000L) == 1)
{
}
}
}
}
/// <summary>
/// Main thread to host the client process.
/// </summary>
/// <param name="threadContext">A ZMQ.Context instance.</param>
public void Run(object threadContext)
{
try
{
Context ctx = (Context)threadContext;
// This would be a using statement on data and ctl but we need to close and reopen sockets, which means
// we have to Dispose() of a socket early and create a new instance. C#, wisely so, prevents you from
// reassigning a using variable.
Socket data = null, ctl = null, disk = null, diskACK = null;
try
{
data = ctx.Socket(SocketType.SUB);
ctl = ctx.Socket(SocketType.REQ);
disk = new Socket(SocketType.PUSH);
diskACK = new Socket(SocketType.PULL);
// Set the HWM for the disk PUSH so that the PUSH blocks if the PULL can't keep up writing:
if (diskHWM > 0) disk.SNDHWM = diskHWM;
//disk.SndBuf = 1048576 * diskHWM * 4;
// inproc:// binds are immediate.
disk.Bind("inproc://disk");
diskACK.Bind("inproc://diskack");
// Create the disk PULL thread:
Thread diskWriterThread = new Thread(new ParameterizedThreadStart(DiskWriterThread));
diskWriterThread.Start(ctx);
data.RCVHWM = networkHWM;
//data.RcvBuf = 1048576 * hwm * 4;
// NOTE: work-around for MS bug in WinXP's networking stack. See http://support.microsoft.com/kb/201213 for details.
data.RcvBuf = 0;
if (tsp == Transport.EPGM || tsp == Transport.PGM)
{
data.Rate = pgmRate;
}
string address = tsp.ToString().ToLower() + "://" + deviceData + ":" + portData;
Console.WriteLine("Connect(\"{0}\")", address);
data.Connect(address);
data.Subscribe(subscription, Encoding.Unicode);
// Connect to the control request socket:
ctl.RcvBuf = 1048576 * 4;
ctl.SndBuf = 1048576 * 4;
address = "tcp://" + deviceCtl + ":" + portCtl.ToString();
Console.WriteLine("Connect(\"{0}\")", address);
ctl.Connect(address);
Guid myIdentity = Guid.NewGuid();
ctl.Identity = new byte[1] { (byte)'@' }.Concat(myIdentity.ToByteArray()).ToArray();
Thread.Sleep(500);
// Begin client logic:
naks = null;
numBytes = 0;
int ackCount = -1;
byte[] nakBuf = null;
bool shuttingDown = false;
Stopwatch recvTimer = Stopwatch.StartNew();
long lastElapsedMilliseconds = recvTimer.ElapsedMilliseconds;
int msgsRecv = 0, msgsWritten = 0;
try
{
// Poll for incoming messages on the data SUB socket:
PollItem[] pollItems = new PollItem[3];
pollItems[0] = data.CreatePollItem(IOMultiPlex.POLLIN);
pollItems[1] = ctl.CreatePollItem(IOMultiPlex.POLLIN /* | IOMultiPlex.POLLOUT */);
pollItems[2] = diskACK.CreatePollItem(IOMultiPlex.POLLIN);
ZMQStateMasheen<DataSUBState> dataFSM;
ZMQStateMasheen<ControlREQState> controlFSM;
Queue<QueuedControlMessage> controlStateQueue = new Queue<QueuedControlMessage>();
DateTimeOffset lastSentNAKs = DateTimeOffset.UtcNow.Subtract(TimeSpan.FromSeconds(60));
// We create a state machine to handle our send/recv state:
dataFSM = new ZMQStateMasheen<DataSUBState>(
// Set the initial state:
DataSUBState.Recv,
// Initial state handler:
new ZMQStateMasheen<DataSUBState>.State(DataSUBState.Recv, (sock, revents) =>
{
Queue<byte[]> packet = sock.RecvAll();
if (packet.Count == 0) { trace("FAIL!"); return DataSUBState.Recv; }
string sub = Encoding.Unicode.GetString(packet.Dequeue());
if (packet.Count == 0) { trace("FAIL!"); return DataSUBState.Recv; }
string cmd = Encoding.Unicode.GetString(packet.Dequeue());
if ((naks != null) && (cmd == "DATA"))
{
if (packet.Count == 0) { trace("FAIL!"); return DataSUBState.Recv; }
int chunkIdx = BitConverter.ToInt32(packet.Dequeue(), 0);
// Count the number of messages received from the network:
++msgsRecv;
if (ChunkReceived != null) ChunkReceived(this, chunkIdx);
// Already received this chunk?
if (!naks[chunkIdx])
{
trace("ALREADY RECV {0}", chunkIdx);
return DataSUBState.Recv;
}
trace("RECV {0}", chunkIdx);
// Queue up the disk writes with PUSH/PULL and an HWM on a separate thread to maintain as
// constant disk write throughput as we can get... The HWM will enforce the PUSHer to block
// when the PULLer cannot receive yet.
if (packet.Count == 0) return DataSUBState.Recv;
byte[] chunk = packet.Dequeue();
disk.SendMore(cmdWrite);
disk.SendMore(BitConverter.GetBytes(chunkIdx));
// This will block if the disk queue gets full:
disk.Send(chunk);
chunk = null;
return DataSUBState.Recv;
}
else if (cmd == "PING")
{
if (shuttingDown) return DataSUBState.Recv;
controlStateQueue.Enqueue(new QueuedControlMessage(ControlREQState.SendALIVE, null));
return DataSUBState.Recv;
}
else
{
return DataSUBState.Recv;
}
})
);
// We create a state machine to handle our send/recv state:
object tmpControlState = null;
controlFSM = new ZMQStateMasheen<ControlREQState>(
ControlREQState.Nothing,
new ZMQStateMasheen<ControlREQState>.State(ControlREQState.Nothing, (sock, revents) =>
{
// Do nothing state.
return ControlREQState.Nothing;
}),
new ZMQStateMasheen<ControlREQState>.State(ControlREQState.SendALIVE, (sock, revents) =>
{
if (shuttingDown) return ControlREQState.Nothing;
ctl.SendMore(ctl.Identity);
ctl.Send("ALIVE", Encoding.Unicode);
return new ZMQStateMasheen<ControlREQState>.MoveOperation(ControlREQState.RecvALIVE, false);
}),
new ZMQStateMasheen<ControlREQState>.State(ControlREQState.RecvALIVE, (sock, revents) =>
{
Queue<byte[]> packet = ctl.RecvAll();
if (packet.Count == 0) return ControlREQState.Nothing;
string cmd = Encoding.Unicode.GetString(packet.Dequeue());
//trace("Server: '{0}'", cmd);
if (cmd == "") return ControlREQState.Nothing;
else if (cmd == "WHOAREYOU") return new ZMQStateMasheen<ControlREQState>.MoveOperation(ControlREQState.SendJOIN, true);
return ControlREQState.Nothing;
}),
new ZMQStateMasheen<ControlREQState>.State(ControlREQState.SendJOIN, (sock, revents) =>
{
if (shuttingDown) return ControlREQState.Nothing;
// Send a JOIN request:
ctl.SendMore(ctl.Identity);
ctl.Send("JOIN", Encoding.Unicode);
return ControlREQState.RecvJOIN;
}),
new ZMQStateMasheen<ControlREQState>.State(ControlREQState.RecvJOIN, (sock, revents) =>
{
Queue<byte[]> packet = ctl.RecvAll();
if (packet.Count == 0) return ControlREQState.Nothing;
string resp = Encoding.Unicode.GetString(packet.Dequeue());
if (resp != "JOINED")
{
// TODO: handle this failure.
trace("FAIL!"); return ControlREQState.Nothing;
}
// TODO: make this atomically succeed or fail without corrupting state
if (packet.Count == 0) { trace("FAIL!"); return ControlREQState.Nothing; }
numChunks = BitConverter.ToInt32(packet.Dequeue(), 0);
numBytes = ((numChunks + 7) & ~7) >> 3;
nakBuf = new byte[numBytes];
if (packet.Count == 0) { trace("FAIL!"); return ControlREQState.Nothing; }
chunkSize = BitConverter.ToInt32(packet.Dequeue(), 0);
if (packet.Count == 0) { trace("FAIL!"); return ControlREQState.Nothing; }
int numFiles = BitConverter.ToInt32(packet.Dequeue(), 0);
List<TarballEntry> tbes = new List<TarballEntry>(numFiles);
for (int i = 0; i < numFiles; ++i)
{
if (packet.Count == 0) { trace("FAIL!"); return ControlREQState.Nothing; }
string fiName = Encoding.Unicode.GetString(packet.Dequeue());
if (packet.Count == 0) { trace("FAIL!"); return ControlREQState.Nothing; }
long length = BitConverter.ToInt64(packet.Dequeue(), 0);
if (packet.Count == 0) { trace("FAIL!"); return ControlREQState.Nothing; }
byte[] hash = packet.Dequeue();
TarballEntry tbe = new TarballEntry(fiName, length, hash);
tbes.Add(tbe);
}
lock (tarballLock)
{
if (tarball != null)
{
tarball.Close();
tarball.Dispose();
tarball = null;
}
}
lock (tarballLock)
{
// Create the tarball reader that writes the files locally:
tarball = new TarballStreamReader(downloadDirectory, tbes);
// Get our local download state:
naks = getClientState(this, tarball);
}
ackCount = naks.Cast<bool>().Take(numChunks).Count(b => !b);
return new ZMQStateMasheen<ControlREQState>.MoveOperation(ControlREQState.SendNAKS, true);
}),
new ZMQStateMasheen<ControlREQState>.State(ControlREQState.SendNAKS, (sock, revents) =>
{
if (shuttingDown) return ControlREQState.Nothing;
if (nakBuf == null) return ControlREQState.Nothing;
// Send our NAKs:
ctl.SendMore(ctl.Identity);
ctl.SendMore("NAKS", Encoding.Unicode);
// TODO: RLE!
naks.CopyTo(nakBuf, 0);
trace("SEND NAK");
ctl.Send(nakBuf);
return ControlREQState.RecvNAKS;
}),
new ZMQStateMasheen<ControlREQState>.State(ControlREQState.RecvNAKS, (sock, revents) =>
{
Queue<byte[]> packet = ctl.RecvAll();
// Don't care what the response is for now.
return ControlREQState.Nothing;
})
);
// Disk-write acknowledgement poll input handler:
pollItems[2].PollInHandler += new PollHandler((Socket sock, IOMultiPlex revents) =>
{
byte[] idxPkt = sock.RecvAll().Dequeue();
int chunkIdx = BitConverter.ToInt32(idxPkt, 0);
naks[chunkIdx] = false;
// Count the number of messages written to disk:
++msgsWritten;
++ackCount;
// Profiled - terrible performance. Not a big surprise here.
//ackCount = naks.Cast<bool>().Take(numChunks).Count(b => !b);
// If we ran up the timer, send more ACKs:
if (DateTimeOffset.UtcNow.Subtract(lastSentNAKs).TotalMilliseconds >= 250d)
{
lastSentNAKs = DateTimeOffset.UtcNow;
controlStateQueue.Enqueue(new QueuedControlMessage(ControlREQState.SendNAKS, null));
}
});
pollItems[0].PollInHandler += new PollHandler(dataFSM.StateMasheen);
pollItems[1].PollInHandler += new PollHandler(controlFSM.StateMasheen);
DateTimeOffset lastRecv = DateTimeOffset.UtcNow;
// Start the control state machine with a JOIN:
controlStateQueue.Enqueue(new QueuedControlMessage(ControlREQState.SendJOIN, null));
// Create a socket poller for the data socket:
while (true)
{
// If we disposed of the previous control socket, create a new one:
if (ctl == null)
{
trace("Creating new CONTROL socket");
// Set up new socket:
ctl = ctx.Socket(SocketType.REQ);
// Connect to the control request socket:
ctl.Connect("tcp://" + deviceCtl + ":" + portCtl.ToString());
ctl.Identity = new byte[1] { (byte)'@' }.Concat(myIdentity.ToByteArray()).ToArray();
}
if (!shuttingDown && IsSendState(controlFSM.CurrentState))
{
// If we ran up the timer, send more NAKs:
if (DateTimeOffset.UtcNow.Subtract(lastSentNAKs).TotalMilliseconds >= 200d)
{
lastSentNAKs = DateTimeOffset.UtcNow;
controlStateQueue.Enqueue(new QueuedControlMessage(ControlREQState.SendNAKS, null));
}
if (controlStateQueue.Count > 0)
{
var msg = controlStateQueue.Dequeue();
// Roll up similar state requests:
int dupes = 0;
while (controlStateQueue.Count > 0)
{
if (controlStateQueue.Peek().NewState != msg.NewState) break;
msg = controlStateQueue.Dequeue();
++dupes;
}
if (dupes > 0)
{
trace("Removed {0} redundant queued states", dupes);
}
tmpControlState = msg.Object;
// Run the control state machine to SEND:
controlFSM.CurrentState = msg.NewState;
controlFSM.StateMasheen(ctl, IOMultiPlex.POLLOUT);
}
}
// Measure our message send rate per minute:
long elapsed = recvTimer.ElapsedMilliseconds - lastElapsedMilliseconds;
if (elapsed >= 100L)
{
NetworkRecvChunksPerMinute = (int)((msgsRecv * 60000L) / elapsed);
DiskWriteChunksPerMinute = (int)((msgsWritten * 60000L) / elapsed);
lastElapsedMilliseconds = recvTimer.ElapsedMilliseconds;
msgsRecv = 0;
msgsWritten = 0;
}
if (!recvTimer.IsRunning)
{
recvTimer.Reset();
recvTimer.Start();
lastElapsedMilliseconds = 0L;
}
if (ackCount >= numChunks)
{
disk.Send(cmdExit);
break;
}
if (ctx.Poll(pollItems, 100L) == 0)
Thread.Sleep(1);
}
// Wait for the disk writer to finish up:
shuttingDown = true;
diskWriterThread.Join();
// If we were last to await some response, receive it:
if (controlFSM.CurrentState == ControlREQState.RecvALIVE || controlFSM.CurrentState == ControlREQState.RecvJOIN || controlFSM.CurrentState == ControlREQState.RecvNAKS)
{
PollItem[] recvPoll = new PollItem[1];
recvPoll[0] = ctl.CreatePollItem(IOMultiPlex.POLLIN);
recvPoll[0].PollInHandler += new PollHandler(controlFSM.StateMasheen);
// Wait a bit to receive the response:
for (int i = 0; (i < 20) && ((ctx.Poll(recvPoll, 100) == 1) && (ctl != null)); ++i)
{
}
}
// Send the LEAVE message:
ctl.SendMore(ctl.Identity);
ctl.Send("LEAVE", Encoding.Unicode);
Completed = true;
}
finally
{
if (tarball != null) tarball.Dispose();
}
}
finally
{
if (data != null) data.Dispose();
if (ctl != null) ctl.Dispose();
}
}
catch (System.Exception ex)
{
Console.Error.WriteLine(ex.ToString());
}
}
public void Write(byte value)
{
var bits = new BitArray(new byte[] { value });
foreach(var b in bits.Cast<bool>())
{
Write(b);
}
}
public static IEnumerable <bool> AsEnumerable(this System.Collections.BitArray source)
{
TypeCheckEnumerable(source, s => s.AsEnumerable(), (s) => s[0]);
return(source.Cast <bool>());
}