/// <summary>
/// Processes a heartbeat message
/// </summary>
/// <param name="reg">The registry received from the remote node</param>
private void ProcessHearbeat(GossipRegistry reg) {
try {
// Merge received registry with local
lock (registryLock) {
foreach (var node in reg.GossipList) {
//If remote node has node suspended, do not add or update it (stale data)
if (node.Status != GossipStatus.Suspended) {
// Get matching node from registry
var localRegNode = registry.GossipList.FirstOrDefault(n => n.NodeId.Equals(node.NodeId));
if (localRegNode == null) {
// Add the node to the local registry
registry.GossipList.Add(node);
if (this.OnNodeAdded != null) {
this.OnNodeAdded(this, node);
}
localRegNode = registry.GossipList.FirstOrDefault(n => n.NodeId.Equals(node.NodeId));
}
// Update heartbeats to lowest values
localRegNode.GossipHeartbeats = Math.Min(localRegNode.GossipHeartbeats, node.GossipHeartbeats);
localRegNode.SuspectMatrix = node.SuspectMatrix;
if (!localRegNode.SuspectMatrix.ContainsKey(localNode.NodeId)) { localRegNode.SuspectMatrix.Add(localNode.NodeId, false); }
// Apply updates if version is newer
if (node.UpdateVersion > localRegNode.UpdateVersion) {
//Update node
if (localRegNode.ListenerAddressList != node.ListenerAddressList) { localRegNode.ListenerAddressList = node.ListenerAddressList; }
if (localRegNode.Partition != node.Partition) { localRegNode.Partition = node.Partition; }
if (localRegNode.HostName != node.HostName) { localRegNode.HostName = node.HostName; }
//Do not update status. Every node must determine status on it's own.
//if (localRegNode.Status != node.Status) { localRegNode.Status = node.Status; }
localRegNode.UpdateVersion = node.UpdateVersion;
}
}
}
}
} catch (Exception ex) {
//TODO: Log Error
}
}
/// <summary>
/// Starts the cluster
/// </summary>
public void Start() {
//Configure listener socket
gossipListenerContext = new ZContext();
gossipListenerSocket = new ZSocket(gossipListenerContext, ZSocketType.ROUTER);
gossipListenerSocket.IdentityString = localNode.NodeId.ToString();
foreach (string e in localNode.ListenerAddressList) {
gossipListenerSocket.Bind(e);
}
//Configure listener thread
listener = new Task(() => {
var poll = ZPollItem.CreateReceiver();
ZMessage incomming;
ZError error;
while (true) {
try {
// Receive
if (gossipListenerSocket.PollIn(poll, out incomming, out error, TimeSpan.FromMilliseconds(64))) {
using (incomming) {
messageQueue.Add(incomming[1].Read());
Debug.WriteLine("Received {0}", incomming[1].ReadString());
}
}
} catch (Exception ex) {
//TODO: Log errors
}
if (listenerCancellation.Token.IsCancellationRequested) {
//listenerSource.Token.ThrowIfCancellationRequested();
break;
}
}
}, listenerCancellation.Token);
queueProcessor = new Task(() => {
while (true) {
try {
byte[] m = messageQueue.Take(queueCancellation.Token);
//First 4 bytes indicates type of message
byte[] buffer = new byte[4];
Array.Copy(m, buffer, 4);
GossipMessageType msgType = (GossipMessageType)BitConverter.ToInt32(buffer, 0);
//Read the rest of the message into a stream
using (MemoryStream ms = new MemoryStream(m, 4, m.Length - 4)) {
//Process message by type
switch (msgType) {
case GossipMessageType.Heartbeat:
try {
//Deserialize message
GossipRegistry reg = Serializer.Deserialize<GossipRegistry>(ms);
//Verify cluster key
if (reg.ClusterKey.Equals(localNode.ClusterKey)) {
ProcessHearbeat(reg);
} else {
//TODO: Log Bad Key
}
} catch (Exception ex) {
//TODO: Log Error
}
break;
case GossipMessageType.Join:
try {
//Deserialize message
GossipNode joinUpdate = Serializer.Deserialize<GossipNode>(ms);
//Verify cluster key
if (joinUpdate.ClusterKey.Equals(localNode.ClusterKey)) {
ProcessJoin(joinUpdate);
} else {
//TODO: Log Bad Key
}
} catch (Exception ex) {
//TODO: Log Error
}
break;
case GossipMessageType.Leave:
try {
//Deserialize message
GossipNode leaveUpdate = Serializer.Deserialize<GossipNode>(ms);
//Verify cluster key
if (leaveUpdate.ClusterKey.Equals(localNode.ClusterKey)) {
ProcessLeave(leaveUpdate);
} else {
//TODO: Log Bad Key
}
} catch (Exception ex) {
//TODO: Log Error
}
break;
}
}
} catch (Exception ex) {
//TODO: Handle exception
}
if (queueCancellation.IsCancellationRequested) {
break;
}
}
}, queueCancellation.Token);
//Start processor
queueProcessor.Start();
//Start listener
listener.Start();
//Check for seed node
bool joinedCluster = false;
foreach (var seedHost in seedAddressList) {
//Attempt to join cluster through seed node
string gossipSeedAddress = string.Format("tcp://{0}:{1}", seedHost.Address.ToString(), seedHost.Port);
Console.WriteLine("Contacting seed node {0}", gossipSeedAddress);
registry = new GossipRegistry() {
ClusterKey = localNode.ClusterKey,
GossipList = new List<GossipNode>()
};
// Join cluster (cluster will add node to registry and return it back)
DateTime joinRequestDate = DateTime.Now;
List<byte> serializedJoinRequest = new List<byte>();
using (MemoryStream ms = new MemoryStream()) {
//Serialize local node
Serializer.Serialize(ms, localNode);
ms.Position = 0;
byte[] serialBuffer = new byte[ms.Length];
ms.Read(serialBuffer, 0, serialBuffer.Length);
serializedJoinRequest.AddRange(BitConverter.GetBytes((int)GossipMessageType.Join));
serializedJoinRequest.AddRange(serialBuffer);
//Send local node to seed
using (var joinContext = new ZContext())
using (var joinSocket = new ZSocket(joinContext, ZSocketType.DEALER)) {
joinSocket.Linger = System.TimeSpan.FromMilliseconds(50);
// Bind
joinSocket.Connect(gossipSeedAddress);
//hbSocket.IdentityString = gossipListenerAddress;
joinSocket.Send(new ZFrame(serializedJoinRequest.ToArray()));
var joinReceiver = ZPollItem.CreateReceiver();
ZMessage joinMessage;
ZError joinError;
//joinSocket.PollIn(joinReceiver, out joinMessage, out joinError, TimeSpan.FromMilliseconds(100));
System.Threading.Thread.Sleep(1);
}
// While we haven't timed out
while (DateTime.Now.Subtract(joinRequestDate).TotalMilliseconds < gossipJoinTimeout) {
//Sleep 1 second and check registry to see if we heard back and we're added.
System.Threading.Thread.Sleep(1000);
if (registry.GossipList.Any(n => n.NodeId.Equals(localNode.NodeId))) {
break;
}
}
}
if (registry.GossipList.Any(n => n.NodeId.Equals(localNode.NodeId))) {
// We heard back from the cluster and we're a member!
joinedCluster = true;
Console.WriteLine("Successfully joined cluster. Starting Gossip.");
break;
} else {
//Join timeout
joinedCluster = false;
Console.WriteLine("Timeout joining cluster. Exiting.");
}
}
//If we didn't join the cluster through the seed node, start a new cluster
if (!joinedCluster) {
Console.WriteLine("Starting new cluster.");
// Start new cluster
registry = new GossipRegistry() {
ClusterKey = localNode.ClusterKey,
GossipList = new List<GossipNode>()
};
registry.GossipList.Add(localNode);
}
//Start gossip timer
t = new System.Timers.Timer(gossipInterval);
t.Elapsed += GossipInterval_Elapsed;
t.Enabled = true;
t.Start();
}
