private void NodeListUpdated(Node node, NodeUpdateAction updateAction)
{
//not interested in any non-active nodes
//other states usually happen when a node comes up online only
if (!(node.NodeState == NodeState.Active))
return;
if (updateAction == NodeUpdateAction.Removed)
_consistentHash.RemoveItem(node.EndPoint.ToString());
else if ((updateAction == NodeUpdateAction.Added) || (updateAction == NodeUpdateAction.Updated))
_consistentHash.StoreItem(node.EndPoint.ToString());
}
public void MoveObjectCallBack(object state)
{
Node targetNode = (Node)state;
string nodeKeyHash = Hasher.GetHash(targetNode.EndPoint.ToString());
string previousNodeHash = Hasher.GetHash(_nodeResolver.GetPreviousItemInCircle(targetNode.EndPoint.ToString()));
//if self is B, typical scenarios -> A-newnode-MID-B, MID-A-newnode-B-C
//A being the previous node, B being the current node and newnode being the new node.
string lowerBound = previousNodeHash;
string upperBound = nodeKeyHash;
IEnumerable<KeyValuePair<string, ServerCacheItem>> candidateList = _cache.Where<KeyValuePair<string, ServerCacheItem>>(x =>
((x.Value.Hash.CompareTo(upperBound) < 0) &&
(x.Value.Hash.CompareTo(lowerBound) > 0)));
//special case: A-MID-newnode-B
if (lowerBound.CompareTo(upperBound) > 0)
{
candidateList = _cache.Where<KeyValuePair<string, ServerCacheItem>>(x =>
((x.Value.Hash.CompareTo(upperBound) < 0) ||
(x.Value.Hash.CompareTo(lowerBound) > 0)));
}
if (candidateList.Count() == 0)
return;
Node self = new Node() { EndPoint = TcpHelper.SelfIPAddress, NodeState= NodeState.Active};
//prepare the connection
string endPoint = string.Format("net.tcp://{0}:{1}/HoCCacheService", targetNode.EndPoint.ToString(), targetNode.ServicePort);
CacheServiceReference.CacheServiceClient nodeService = new CacheServiceReference.CacheServiceClient(new NetTcpBinding(SecurityMode.None), new EndpointAddress(endPoint));
try
{
//move each of the object to the other node async
foreach (KeyValuePair<string, ServerCacheItem> valuePair in candidateList)
{
nodeService.DoInterNodeObjectTransfer(self, valuePair.Key, valuePair.Value);
valuePair.Value.ItemState = CacheItemState.Moved;
valuePair.Value.RelocatedTo = targetNode;
};
nodeService.EndInterNodeObjectTransfer(self);
Thread.Sleep(2000);//wait the current thread - note that this is not the main thread
RemoveOrResetMovedItems(targetNode);
//note that any object update that happens during this time is handled by the CacheService.
//basically it copies the update to the new node too.
}
catch
{
ResetMovedObjectState(); //in case of any issues revert back to normal state.
}
}
public bool DoInterNodeObjectTransfer(Node sourceNode, string key, ServerCacheItem cacheItem)
{
//note that DoInterNodeObjectTransfer could be called during a shutdown of the sourceNode too
//the below case happens only when this current node comes online
if (NodeState == Common.NodeState.WaitingForNeighbourNode)
{
NodeState = NodeState.ReceivingFromOtherNode;
heartBeatResetter.Change(6000, Timeout.Infinite); //keep resetting the timer
}
Func<string, ServerCacheItem, ServerCacheItem> updateValueFactory = ((x, y) => (cacheItem));
_localCache.AddOrUpdate(key, cacheItem, updateValueFactory);
return true;
}
public void StoreCacheItem(string key, ClientCacheItem value)
{
ServerCacheItem cacheItemWrapped;
if (!_localCache.TryGetValue(key, out cacheItemWrapped))
cacheItemWrapped = new ServerCacheItem() { ItemState = CacheItemState.None };
cacheItemWrapped.Hash = Hasher.GetHash(key);
cacheItemWrapped.Value = value;
Func<string, ServerCacheItem, ServerCacheItem> updateValueFactory = ((x, y) => (cacheItemWrapped));
_localCache.AddOrUpdate(key, cacheItemWrapped, updateValueFactory);
//has the item been moved to another node?
if (cacheItemWrapped.ItemState == CacheItemState.Moved)
{
Debug.Assert(cacheItemWrapped.RelocatedTo != null);
Node self = new Node() { EndPoint = TcpHelper.SelfIPAddress, NodeState = NodeState.Active };
//prepare the connection
string endPoint = string.Format("net.tcp://{0}:{1}/HoCCacheService", cacheItemWrapped.RelocatedTo.EndPoint.ToString(), cacheItemWrapped.RelocatedTo.ServicePort);
CacheServiceReference.CacheServiceClient nodeService = new CacheServiceReference.CacheServiceClient(new NetTcpBinding(SecurityMode.None), new EndpointAddress(endPoint));
//update to the target node.
nodeService.StoreCacheItem(key, value);
}
}
public bool EndInterNodeObjectTransfer(Node sourceNode)
{
NodeState = NodeState.Active; //get online
_heartBeat.SendBeat();//force a beat now
heartBeatResetter.Change(0, Timeout.Infinite);//dont want the timer to fire henceforth
return true;
}
private void RemoveOrResetMovedItems(Node targetNode)
{
Func<Node, bool> nodeCompare = x => (x.EndPoint.ToString().CompareTo(targetNode.EndPoint.ToString()) == 0);
Node targetNodeInternal = _nodeTracker.ActiveNodes.First<Node>(nodeCompare);
if (targetNodeInternal == null)
ResetMovedObjectState();
else if (targetNodeInternal.NodeState == NodeState.Active) //is the target node now active?
RemoveMovedObjectState();
else
ResetMovedObjectState(); // reset all moved states as the node does not appear to have got online yet
}
private void NodeListUpdated(Node node, NodeUpdateAction updateAction)
{
//has the node just come online and waiting for its data that I have?
if ((updateAction == NodeUpdateAction.Added) && (node.NodeState == NodeState.WaitingForNeighbourNode))
{
//was that node me?
if (node.EndPoint.ToString().CompareTo(TcpHelper.SelfIPAddress.ToString()) == 0)
return;
bool nodeIsNeighbourBehind = NodeIsNeighbourBehind(node);
if (!nodeIsNeighbourBehind)
return;
ThreadPool.QueueUserWorkItem(MoveObjectCallBack, node);
}
}
private bool NodeIsNeighbourBehind(Node node)
{
//add the new node to a temporary clone of the active consistent hash
ConsistentHash consistentHashTrialRun = _nodeResolver.GetActiveConsistentHashClone();
consistentHashTrialRun.StoreItem(node.EndPoint.ToString());
//ask the consistent hash that if the node was around, would that node be my immediate previous neighbour?
string previousItem = consistentHashTrialRun.GetPreviousItemInCircle(TcpHelper.SelfIPAddress.ToString());
if (node.EndPoint.ToString().CompareTo(previousItem) == 0)
return true;
return false;
}
public void PerformShutdownMove()
{
//targetnode is the one which comes next in the circle, effectively returned by GetObjectLocation
Node targetNode = _nodeResolver.GetObjectLocation(TcpHelper.SelfIPAddress.ToString());
//is targetnode self? (am I the only node in the circle?)
if (targetNode.EndPoint.ToString().CompareTo(TcpHelper.SelfIPAddress.ToString()) == 0)
return;
string endPoint = string.Format("net.tcp://{0}:{1}/HoCCacheService", targetNode.EndPoint.ToString(), targetNode.ServicePort);
CacheServiceReference.CacheServiceClient nodeService = new CacheServiceReference.CacheServiceClient(new NetTcpBinding(SecurityMode.None), new EndpointAddress(endPoint));
Node self = new Node() { EndPoint = TcpHelper.SelfIPAddress, NodeState= NodeState.Active};
//move the objects to the nearest node async
Parallel.ForEach<KeyValuePair<string, ServerCacheItem>>(_cache, valuePair =>
{
nodeService.DoInterNodeObjectTransfer(self, valuePair.Key, valuePair.Value);
valuePair.Value.ItemState = CacheItemState.Moved;
valuePair.Value.RelocatedTo = targetNode;
});
}
private void HeartBeatTracker(HeartBeatData heartBeatData)
{
IPAddress nodeIPAddress = new IPAddress(heartBeatData.IPAddress);
Node node = _activeNodes.Find(x => x.EndPoint.ToString() == nodeIPAddress.ToString());
NodeUpdateAction updateAction;
if (node != null)
{
//update the last heard time, node state
node.HeartBeatLastHeardAt = DateTime.Now;
node.NodeState = heartBeatData.NodeState;
node.EndPoint = nodeIPAddress;
node.ServicePort = heartBeatData.ServicePort;
updateAction = NodeUpdateAction.Updated;
}
else
{
node = new Node()
{
EndPoint = nodeIPAddress,
ServicePort = heartBeatData.ServicePort,
HeartBeatLastHeardAt = DateTime.Now,
NodeState = heartBeatData.NodeState
};
lock (_activeNodes)
{
_activeNodes.Add(node); //add the new node which appears to have just come online.
}
updateAction = NodeUpdateAction.Added;
}
if (OnNodeListUpdated != null)
OnNodeListUpdated(node, updateAction);
}
private void NodeListUpdated(Node node, NodeUpdateAction updateAction)
{
if (updateAction == NodeUpdateAction.Added)
{
Dispatcher.Invoke((Action)(() => {
tvNodes.Items.Add(node.EndPoint.ToString());
if (tvNodes.Items.Count == 1)
{
SetSelectedItem(tvNodes, tvNodes.Items[0]);
}
UpdateStatusMessage("Discovered new node " + node.EndPoint.ToString());
}));
}
else if (updateAction == NodeUpdateAction.Removed)
{
Dispatcher.Invoke((Action)(() => {
tvNodes.Items.Remove(node.EndPoint.ToString());
UpdateStatusMessage("Removed dead node " + node.EndPoint.ToString());
ClearGrid();
}));
}
}