/// <summary>
/// Maintenance task to stabilize the local node's predecessor as per the Chord paper.
/// </summary>
/// <param name="sender">The backgroundworker thread that this task is running on.</param>
/// <param name="ea">Args (ignored)</param>
private void StabilizePredecessors(object sender, DoWorkEventArgs ea)
{
BackgroundWorker me = (BackgroundWorker)sender;
while (!me.CancellationPending)
{
if (this.Predecessor != null)
{
try
{
// validate predecessor (in case of error, predecessor becomes null
// and is fixed by stabilizesuccessors and notify.
ChordInstance instance = ChordServer.GetInstance(this.Predecessor);
if (!ChordServer.IsInstanceValid(instance))
{
this.Predecessor = null;
}
}
catch (Exception e)
{
ChordServer.Log(LogLevel.Error, "StabilizePredecessors", "StabilizePredecessors error: {0}", e.Message);
this.Predecessor = null;
}
}
// TODO: make this configurable either via config file or passed in via arguments.
Thread.Sleep(5000);
}
}
/// <summary>
/// Depart from the Chord ring and stop maintenance.
/// </summary>
public void Depart()
{
// first, stop maintenance so nothing gets broken
StopMaintenance();
try
{
// courteously introduce departing node's successor and predecessor
// ...successor, meet predecessor; predecessor, meet successor
ChordInstance instance = ChordServer.GetInstance(this.Successor);
instance.Predecessor = this.Predecessor;
instance = ChordServer.GetInstance(this.Predecessor);
instance.Successor = this.Successor;
}
catch (Exception e)
{
ChordServer.Log(LogLevel.Error, "Navigation", "Error on Depart ({0}).", e.Message);
}
finally
{
// set local state in such a way as to force it out of the Chord ring
this.Successor = ChordServer.LocalNode;
this.Predecessor = ChordServer.LocalNode;
this.FingerTable = new ChordFingerTable(ChordServer.LocalNode);
for (int i = 0; i < this.SuccessorCache.Length; i++)
{
this.SuccessorCache[i] = ChordServer.LocalNode;
}
}
}
/// <summary>
/// Retrieve the string value for a given ulong
/// key.
/// </summary>
/// <param name="key">The key whose value should be returned.</param>
/// <returns>The string value for the given key, or an empty string if not found.</returns>
public string FindKey(ulong key)
{
// determine the owning node for the key
ChordNode owningNode = ChordServer.CallFindSuccessor(key);
if (owningNode != ChordServer.LocalNode)
{
// if this is not the owning node, call
// FindKey on the remote owning node
return(ChordServer.CallFindKey(owningNode, key));
}
else
{
// if this is the owning node, check
// to see if the key exists in the data store
if (this.m_DataStore.ContainsKey(key))
{
// if the key exists, return the value
return(this.m_DataStore[key]);
}
else
{
// if the key does not exist, return empty string
return(string.Empty);
}
}
}
/// <summary>
/// Replicate the local data store on a background thread.
/// </summary>
/// <param name="sender">The background worker thread this task is running on.</param>
/// <param name="ea">Args (ignored).</param>
private void ReplicateStorage(object sender, DoWorkEventArgs ea)
{
BackgroundWorker me = (BackgroundWorker)sender;
while (!me.CancellationPending)
{
try
{
// replicate each key to the successor safely
foreach (ulong key in this.m_DataStore.Keys)
{
// if the key is local (don't copy replicas)
if (ChordServer.IsIDInRange(key, this.ID, this.Successor.ID))
{
ChordServer.CallReplicateKey(this.Successor, key, this.m_DataStore[key]);
}
}
}
catch (Exception e)
{
// (overly safe here)
ChordServer.Log(LogLevel.Error, "Maintenance", "Error occured during ReplicateStorage ({0})", e.Message);
}
// TODO: make this configurable via config file or passed in as an argument
Thread.Sleep(30000);
}
}
/// <summary>
/// Safely register the TcpChannel for this service.
/// </summary>
/// <param name="port">The port on which the service will listen.</param>
/// <returns>true if registration succeeded; false, otherwise.</returns>
public static bool RegisterService(int port)
{
try
{
if (s_ChordTcpChannel != null)
{
ChordServer.UnregisterService();
}
BinaryServerFormatterSinkProvider provider = new BinaryServerFormatterSinkProvider();
provider.TypeFilterLevel = System.Runtime.Serialization.Formatters.TypeFilterLevel.Full;
IDictionary props = new Hashtable();
props["port"] = port;
s_ChordTcpChannel = new TcpChannel(props, null, provider);
ChannelServices.RegisterChannel(s_ChordTcpChannel, false);
RemotingConfiguration.RegisterWellKnownServiceType(typeof(ChordInstance), "chord", WellKnownObjectMode.Singleton);
}
catch (Exception e)
{
ChordServer.Log(LogLevel.Error, "Configuration", "Unable to register Chord Service ({0}).", e.Message);
return(false);
}
ChordServer.Log(LogLevel.Info, "Configuration", "Chord Service registered on port {0}.", port);
return(true);
}
/// <summary>
/// Maintenance task to ensure that the local node has valid successor node. Roughly equivalent
/// to what is called out in the Chord paper.
/// </summary>
/// <param name="sender">The worker thread the task is running on.</param>
/// <param name="ea">Args (ignored here).</param>
private void StabilizeSuccessors(object sender, DoWorkEventArgs ea)
{
BackgroundWorker me = (BackgroundWorker)sender;
while (!me.CancellationPending)
{
try
{
// check in successor and if it's bad, replace it with
// the next live entry in the successor cache
ChordNode succPredNode = ChordServer.GetPredecessor(this.Successor);
if (succPredNode != null)
{
if (ChordServer.IsIDInRange(succPredNode.ID, this.ID, this.Successor.ID))
{
this.Successor = succPredNode;
}
// ignoring return because bad node will be detected on next invocation
ChordServer.CallNotify(this.Successor, ChordServer.LocalNode);
GetSuccessorCache(this.Successor);
}
else
{
bool successorCacheHelped = false;
foreach (ChordNode entry in this.m_SuccessorCache)
{
ChordInstance instance = ChordServer.GetInstance(entry);
if (ChordServer.IsInstanceValid(instance))
{
this.Successor = entry;
ChordServer.CallNotify(this.Successor, ChordServer.LocalNode);
GetSuccessorCache(this.Successor);
successorCacheHelped = true;
break;
}
}
// if we get here, then we got no help and have no other recourse than to re-join using the initial seed...
if (!successorCacheHelped)
{
ChordServer.Log(LogLevel.Error, "StabilizeSuccessors", "Ring consistency error, Re-Joining Chord ring.");
Join(this.m_SeedNode, this.Host, this.Port);
return;
}
}
}
catch (Exception e)
{
ChordServer.Log(LogLevel.Error, "Maintenance", "Error occured during StabilizeSuccessors ({0})", e.Message);
}
// TODO: this could be tweaked and/or made configurable elsewhere or passed in as arguments
Thread.Sleep(5000);
}
}
/// <summary>
/// Get the successor cache from a remote node and assign an altered version the local successorCache.
/// Gets the remote successor cache, prepends remoteNode and lops off the last entry from the remote
/// successorcache.
/// </summary>
/// <param name="remoteNode">The remote node to get the succesorCache from.</param>
private void GetSuccessorCache(ChordNode remoteNode)
{
ChordNode[] remoteSuccessorCache = ChordServer.GetSuccessorCache(remoteNode);
if (remoteSuccessorCache != null)
{
this.SuccessorCache[0] = remoteNode;
for (int i = 1; i < this.SuccessorCache.Length; i++)
{
this.SuccessorCache[i] = remoteSuccessorCache[i - 1];
}
}
}
/// <summary>
/// Maintenance task to perform ring consistency checking and re-joining to keep a Chord
/// ring stable under extreme churn and in cases of ring damage.
/// </summary>
/// <param name="sender">The calling backgroundworker.</param>
/// <param name="ea">Args (ignored for this task).</param>
private void ReJoin(object sender, DoWorkEventArgs ea)
{
BackgroundWorker me = (BackgroundWorker)sender;
while (!me.CancellationPending)
{
try
{
// if this is the first iteration, then the core logic
// is skipped, as the first iteration generally occurs
// right after node Join - allowing a short buffer for
// routing structures to stabilize improves the utility
// of the ReJoin facility.
if (this.m_HasReJoinRun)
{
// first find the successor for the seed node
if (this.m_SeedNode != null)
{
ChordNode seedSuccessor = FindSuccessor(this.m_SeedNode.ID);
// if the successor is not equal to the seed node, something is fishy
if (seedSuccessor.ID != this.m_SeedNode.ID)
{
// if the seed node is still active, re-join the ring to the seed node
ChordInstance instance = ChordServer.GetInstance(this.m_SeedNode);
if (ChordServer.IsInstanceValid(instance))
{
ChordServer.Log(LogLevel.Error, "ReJoin", "Unable to contact initial seed node {0}. Re-Joining...", this.m_SeedNode);
Join(this.m_SeedNode, this.Host, this.Port);
}
// otherwise, in the future, there will be a cache of seed nodes to check/join from...
// as it may be the case that the seed node simply has disconnected from the network.
}
}
}
else
{
// subsequent iterations will go through the core logic
this.m_HasReJoinRun = true;
}
}
catch (Exception e)
{
ChordServer.Log(LogLevel.Error, "Maintenance", "Error occured during ReJoin ({0})", e.Message);
}
// TODO: the delay between iterations, here, is configurable, and
// ideally should be retrieved from configuration or otherwise passed in...
Thread.Sleep(30000);
}
}
/// <summary>
/// Find the node that is the rightful owner of a given id.
/// </summary>
/// <param name="id">The id whose successor should be found.</param>
/// <param name="hopCount">The number of network hops taken in finding the successor.</param>
/// <returns>The ChordNode that is the Successor of a given ID value.</returns>
public ChordNode FindSuccessor(UInt64 id, int hopCountIn, out int hopCountOut)
{
// is the local node's successor the rightful owner?
if (ChordServer.IsIDInRange(id, this.ID, this.Successor.ID))
{
hopCountOut = hopCountIn;
return(this.Successor);
}
else
{
// otherwise, find the nearest preceding finger, and ask that node.
ChordNode predNode = FindClosestPrecedingFinger(id);
return(ChordServer.CallFindSuccessor(predNode, id, 0, ++hopCountIn, out hopCountOut));
}
}
/// <summary>
/// Called by the predecessor to a remote node, this acts as a dual heartbeat mechanism and more importantly
/// notification mechanism between predecessor and successor.
/// </summary>
/// <param name="node">A ChordNode instance indicating who the calling node (predecessor) is.</param>
public void Notify(ChordNode node)
{
// if the node has absolutely no predecessor, take
// the first one it finds
if (this.Predecessor == null)
{
this.Predecessor = node;
return;
}
// otherwise, ensure that the predecessor that is calling in
// is indeed valid...
if (ChordServer.IsIDInRange(node.ID, this.Predecessor.ID, this.ID))
{
this.Predecessor = node;
return;
}
}
/// <summary>
/// Get a (local or remote) ChordInstance given a ChordNode.
/// </summary>
/// <param name="node">The ChordNode specifying the node to get an instance of.</param>
/// <returns>A ChordInstance from the specified node, or null if an error is encountered.</returns>
public static ChordInstance GetInstance(ChordNode node)
{
if (node == null)
{
ChordServer.Log(LogLevel.Error, "Navigation", "Invalid Node ({0}).", "Null Argument.");
return(null);
}
try
{
ChordInstance retInstance = (ChordInstance)Activator.GetObject(typeof(ChordInstance), string.Format("tcp://{0}:{1}/chord", node.Host, node.PortNumber));
return(retInstance);
}
catch (Exception e)
{
// perhaps instead we should just pass on the error?
ChordServer.Log(LogLevel.Error, "Navigation", "Unable to activate remote server {0}:{1} ({2}).", node.Host, node.PortNumber, e.Message);
return(null);
}
}
/// <summary>
/// Returns the closest successor preceding id.
/// </summary>
/// <param name="id">The id for which the closest finger should be found</param>
/// <returns>The successor node of the closest finger to id in the current node's finger table</returns>
private ChordNode FindClosestPrecedingFinger(UInt64 id)
{
// iterate downward through the finger table looking for the right finger in the right range. if the finger is
// in the range but not valid, keep moving. if the entire finger table is checked without success, check the successor
// cache - if that fails, return the local node as the closest preceding finger.
for (int i = this.FingerTable.Length - 1; i >= 0; i--)
{
// if the finger is more closely between the local node and id and that finger corresponds to a valid node, return the finger
if (this.FingerTable.Successors[i] != null && this.FingerTable.Successors[i] != ChordServer.LocalNode)
{
if (ChordServer.FingerInRange(this.FingerTable.Successors[i].ID, this.ID, id))
{
ChordInstance instance = ChordServer.GetInstance(this.FingerTable.Successors[i]);
if (ChordServer.IsInstanceValid(instance))
{
return(this.FingerTable.Successors[i]);
}
}
}
}
// at this point, not even the successor is any good so go through the successor cache and run the same test
for (int i = 0; i < this.SuccessorCache.Length; i++)
{
if (this.SuccessorCache[i] != null && this.SuccessorCache[i] != ChordServer.LocalNode)
{
if (ChordServer.FingerInRange(this.SuccessorCache[i].ID, this.ID, id))
{
ChordInstance instance = ChordServer.GetInstance(this.SuccessorCache[i]);
if (ChordServer.IsInstanceValid(instance))
{
return(this.SuccessorCache[i]);
}
}
}
}
// otherwise, if there is nothing closer, the local node is the closest preceding finger
return(ChordServer.LocalNode);
}
/// <summary>
/// Calls AddKey remotely.
/// </summary>
/// <param name="remoteNode">The remote node on which to call AddKey.</param>
/// <param name="value">The string value to add.</param>
/// <param name="retryCount">The number of retries to attempt.</param>
public static void CallAddKey(ChordNode remoteNode, string value, int retryCount)
{
ChordInstance instance = ChordServer.GetInstance(remoteNode);
try
{
instance.AddKey(value);
}
catch (System.Exception ex)
{
ChordServer.Log(LogLevel.Debug, "Remote Invoker", "CallAddKey error: {0}", ex.Message);
if (retryCount > 0)
{
CallAddKey(remoteNode, value, --retryCount);
}
else
{
ChordServer.Log(LogLevel.Debug, "Remote Invoker", "CallAddKey failed - error: {0}", ex.Message);
}
}
}
/// <summary>
/// Gets the remote Successor property, given a custom retry count.
/// </summary>
/// <param name="remoteNode">The remote node from which to access the property.</param>
/// <param name="retryCount">The number of times to retry the operation in case of error.</param>
/// <returns>The remote successor, or NULL in case of error.</returns>
public static ChordNode GetSuccessor(ChordNode remoteNode, int retryCount)
{
ChordInstance instance = ChordServer.GetInstance(remoteNode);
try
{
return(instance.Successor);
}
catch (System.Exception ex)
{
ChordServer.Log(LogLevel.Debug, "Remote Accessor", "GetSuccessor error: {0}", ex.Message);
if (retryCount > 0)
{
return(GetSuccessor(remoteNode, --retryCount));
}
else
{
return(null);
}
}
}
/// <summary>
/// Calls FindKey remotely.
/// </summary>
/// <param name="remoteNode">The remote node on which to call FindKey.</param>
/// <param name="key">The key to look up.</param>
/// <param name="retryCount">The number of retries to attempt.</param>
/// <returns>The value corresponding to the key, or empty string if not found.</returns>
public static string CallFindKey(ChordNode remoteNode, ulong key, int retryCount)
{
ChordInstance instance = ChordServer.GetInstance(remoteNode);
try
{
return(instance.FindKey(key));
}
catch (System.Exception ex)
{
ChordServer.Log(LogLevel.Debug, "Remote Invoker", "CallFindKey error: {0}", ex.Message);
if (retryCount > 0)
{
return(CallFindKey(remoteNode, key, --retryCount));
}
else
{
ChordServer.Log(LogLevel.Debug, "Remote Invoker", "CallFindKey failed - error: {0}", ex.Message);
return(string.Empty);
}
}
}
/// <summary>
/// Calls FindSuccessor() remotely, using a default retry value of three.
/// </summary>
/// <param name="remoteNode">The remote node on which to call FindSuccessor().</param>
/// <param name="id">The ID to look up.</param>
/// <param name="retryCount">The number of times to retry the operation in case of error.</param>
/// <param name="hopCountIn">The known hopcount prior to calling FindSuccessor on this node.</param>
/// <param name="hopCountOut">The total hopcount of this operation (either returned upwards, or reported for hopcount efficiency validation).</param>
/// <returns>The Successor of ID, or NULL in case of error.</returns>
public static ChordNode CallFindSuccessor(ChordNode remoteNode, UInt64 id, int retryCount, int hopCountIn, out int hopCountOut)
{
ChordInstance instance = ChordServer.GetInstance(remoteNode);
try
{
return(instance.FindSuccessor(id, hopCountIn, out hopCountOut));
}
catch (System.Exception ex)
{
ChordServer.Log(LogLevel.Debug, "Remote Invoker", "CallFindSuccessor error: {0}", ex.Message);
if (retryCount > 0)
{
return(CallFindSuccessor(remoteNode, id, --retryCount, hopCountIn, out hopCountOut));
}
else
{
hopCountOut = hopCountIn;
return(null);
}
}
}
/// <summary>
/// Calls Notify() remotely, using a default retry value of three.
/// </summary>
/// <param name="remoteNode">The remote on which to call the method.</param>
/// <param name="callingNode">The node to inform the remoteNode of.</param>
/// <param name="retryCount">The number of times to retry the operation in case of error.</param>
/// <returns>True if succeeded, FALSE otherwise.</returns>
public static bool CallNotify(ChordNode remoteNode, ChordNode callingNode, int retryCount)
{
ChordInstance instance = ChordServer.GetInstance(remoteNode);
try
{
instance.Notify(callingNode);
return(true);
}
catch (System.Exception ex)
{
ChordServer.Log(LogLevel.Debug, "Remote Invoker", "CallNotify error: {0}", ex.Message);
if (retryCount > 0)
{
return(CallNotify(remoteNode, callingNode, --retryCount));
}
else
{
return(false);
}
}
}
/// <summary>
/// Update the local node's finger table entries on a background thread.
/// </summary>
/// <param name="sender">The background worker thread this task is running on.</param>
/// <param name="ea">Args (ignored).</param>
private void UpdateFingerTable(object sender, DoWorkEventArgs ea)
{
BackgroundWorker me = (BackgroundWorker)sender;
while (!me.CancellationPending)
{
try
{
// update the fingers moving outwards - once the last finger
// has been reached, start again closest to LocalNode (0).
if (this.m_NextFingerToUpdate >= this.m_FingerTable.Length)
{
this.m_NextFingerToUpdate = 0;
}
try
{
// Node validity is checked by findSuccessor
this.FingerTable.Successors[this.m_NextFingerToUpdate] = FindSuccessor(this.FingerTable.StartValues[this.m_NextFingerToUpdate]);
}
catch (Exception e)
{
ChordServer.Log(LogLevel.Error, "Navigation", "Unable to update Successor for start value {0} ({1}).", this.FingerTable.StartValues[this.m_NextFingerToUpdate], e.Message);
}
this.m_NextFingerToUpdate = this.m_NextFingerToUpdate + 1;
}
catch (Exception e)
{
// (overly safe here)
ChordServer.Log(LogLevel.Error, "Maintenance", "Error occured during UpdateFingerTable ({0})", e.Message);
}
// TODO: make this configurable via config file or passed in as an argument
Thread.Sleep(1000);
}
}
/// <summary>
/// Add a key to the store. Gets a hash of the key, determines
/// the correct owning node, and stores the string value
/// on that node.
/// </summary>
/// <param name="value">The value to add.</param>
public void AddKey(string value)
{
// the key is the hash of the value to
// add to the store, and determines the
// owning NChord node
ulong key = ChordServer.GetHash(value);
// using the local node, determine the correct owning
// node for the data to be stored given the key value
ChordNode owningNode = ChordServer.CallFindSuccessor(key);
if (owningNode != ChordServer.LocalNode)
{
// if this is not the owning node, then call AddKey
// on the actual owning node
ChordServer.CallAddKey(owningNode, value);
}
else
{
// if this is the owning node, then add the
// key to the local data store
this.m_DataStore.Add(key, value);
}
}
/// <summary>
/// Join the current ChordInstance to a chord ring given a seed node.
/// </summary>
/// <param name="seed">Remote node to connect to that is a member of a Chord ring. To start a new Chord ring, a null seed may be provided.</param>
/// <param name="host">The local host name.</param>
/// <param name="port">The tcp port on which this node will listen for calls from other Chord nodes.</param>
/// <returns>true if the Join succeeds; false, otherwise.</returns>
public bool Join(ChordNode seed, string host, int port)
{
// LocalNode is established first to ensure proper logging.
ChordServer.LocalNode = new ChordNode(host, port);
this.m_HasReJoinRun = false;
this.m_SeedNode = seed; // cache the seed node so the ring may re-form itself in case of partition or damage
// safely establish finger table (startvalues and successors, using LocalNode as the default successor)
this.FingerTable = new ChordFingerTable(ChordServer.LocalNode);
// establish successor cache initially with all entries pointing loally
this.SuccessorCache = new ChordNode[3]; // TODO: make this configurable
for (int i = 0; i < this.SuccessorCache.Length; i++)
{
this.SuccessorCache[i] = ChordServer.LocalNode;
}
if (seed != null) // join an existing chord ring
{
ChordServer.Log(LogLevel.Info, "Navigation", "Joining Ring @ {0}:{1}.", seed.Host, seed.PortNumber);
// first, establish successor via seed node
ChordInstance instance = ChordServer.GetInstance(seed);
if (ChordServer.IsInstanceValid(instance))
{
try
{
this.Successor = instance.FindSuccessor(this.ID); // NOTE: this conceivably could be replaced with ChordServer.FindSuccessor()
// disabled: a clever trick that requires only one remote network call is to
// append the successor's successor cache (minus its last entry) to the local
// successor cache, starting at the second entry in the local successor cache.
// during churn, this can break down, so instead the successor cache is populated
// and maintained lazily by maintenance.
// as the successor cache was initialized with the LocalNode as the default
// instance values, "misses" on successor cache entries are gracefully handled by
// simply being forwarded (via the local node) on to the successor node where
// better information may be found and utilized.
//
//this.SuccessorCache = ChordServer.GetSuccessorCache(this.Successor);
}
catch (Exception e)
{
ChordServer.Log(LogLevel.Error, "Navigation", "Error setting Successor Node ({0}).", e.Message);
return(false);
}
}
else
{
ChordServer.Log(LogLevel.Error, "Navigation", "Invalid Node Seed.");
return(false);
}
}
else // start a new ring
{
// not much needs to happen - successor is already established as
// ChordServer.LocalNode,everything else takes place lazily as part of maintenance
ChordServer.Log(LogLevel.Info, "Navigation", "Starting Ring @ {0}:{1}.", this.Host, this.Port);
}
// everything that needs to be populated or kept up-to-date
// lazily is handled via background maintenance threads running periodically
StartMaintenance();
return(true);
}
