/// <summary>
/// Retrieve the string value for a given ulong
/// key.
/// </summary>
/// <param name="key">The key whose value should be returned.</param>
/// <returns>The string value for the given key, or an empty string if not found.</returns>
public string FindKey(ulong key)
{
// determine the owning node for the key
ChordNode owningNode = ChordServer.CallFindSuccessor(key);
if (owningNode != ChordServer.LocalNode)
{
// if this is not the owning node, call
// FindKey on the remote owning node
return(ChordServer.CallFindKey(owningNode, key));
}
else
{
// if this is the owning node, check
// to see if the key exists in the data store
if (this.m_DataStore.ContainsKey(key))
{
// if the key exists, return the value
return(this.m_DataStore[key]);
}
else
{
// if the key does not exist, return empty string
return(string.Empty);
}
}
}
/// <summary>
/// Creates a new instance of the finger table
/// </summary>
/// <param name="nodeIn">The node to populate the finger table with</param>
public ChordFingerTable(ChordNode seed)
{
// populate the start array and successors
for (int i = 0; i < this.Length; i++)
{
this.StartValues[i] = (seed.ID + (UInt64)Math.Pow(2, i)) % UInt64.MaxValue;
this.Successors[i] = seed;
}
}
/// <summary>
/// Maintenance task to ensure that the local node has valid successor node. Roughly equivalent
/// to what is called out in the Chord paper.
/// </summary>
/// <param name="sender">The worker thread the task is running on.</param>
/// <param name="ea">Args (ignored here).</param>
private void StabilizeSuccessors(object sender, DoWorkEventArgs ea)
{
BackgroundWorker me = (BackgroundWorker)sender;
while (!me.CancellationPending)
{
try
{
// check in successor and if it's bad, replace it with
// the next live entry in the successor cache
ChordNode succPredNode = ChordServer.GetPredecessor(this.Successor);
if (succPredNode != null)
{
if (ChordServer.IsIDInRange(succPredNode.ID, this.ID, this.Successor.ID))
{
this.Successor = succPredNode;
}
// ignoring return because bad node will be detected on next invocation
ChordServer.CallNotify(this.Successor, ChordServer.LocalNode);
GetSuccessorCache(this.Successor);
}
else
{
bool successorCacheHelped = false;
foreach (ChordNode entry in this.m_SuccessorCache)
{
ChordInstance instance = ChordServer.GetInstance(entry);
if (ChordServer.IsInstanceValid(instance))
{
this.Successor = entry;
ChordServer.CallNotify(this.Successor, ChordServer.LocalNode);
GetSuccessorCache(this.Successor);
successorCacheHelped = true;
break;
}
}
// if we get here, then we got no help and have no other recourse than to re-join using the initial seed...
if (!successorCacheHelped)
{
ChordServer.Log(LogLevel.Error, "StabilizeSuccessors", "Ring consistency error, Re-Joining Chord ring.");
Join(this.m_SeedNode, this.Host, this.Port);
return;
}
}
}
catch (Exception e)
{
ChordServer.Log(LogLevel.Error, "Maintenance", "Error occured during StabilizeSuccessors ({0})", e.Message);
}
// TODO: this could be tweaked and/or made configurable elsewhere or passed in as arguments
Thread.Sleep(5000);
}
}
/// <summary>
/// Compares ChordNodes on their ID hash value.
/// </summary>
/// <param name="obj">The object to compare this ChordNode to.</param>
/// <returns>A negative value if the object is smaller; zero if they are equal; positive if the object is larger.</returns>
public int CompareTo(object obj)
{
if (obj is ChordNode)
{
ChordNode node = (ChordNode)obj;
return(this.ID.CompareTo(node.ID));
}
throw new ArgumentException("Object is not a ChordNode.");
}
/// <summary>
/// Get the successor cache from a remote node and assign an altered version the local successorCache.
/// Gets the remote successor cache, prepends remoteNode and lops off the last entry from the remote
/// successorcache.
/// </summary>
/// <param name="remoteNode">The remote node to get the succesorCache from.</param>
private void GetSuccessorCache(ChordNode remoteNode)
{
ChordNode[] remoteSuccessorCache = ChordServer.GetSuccessorCache(remoteNode);
if (remoteSuccessorCache != null)
{
this.SuccessorCache[0] = remoteNode;
for (int i = 1; i < this.SuccessorCache.Length; i++)
{
this.SuccessorCache[i] = remoteSuccessorCache[i - 1];
}
}
}
/// <summary>
/// Maintenance task to perform ring consistency checking and re-joining to keep a Chord
/// ring stable under extreme churn and in cases of ring damage.
/// </summary>
/// <param name="sender">The calling backgroundworker.</param>
/// <param name="ea">Args (ignored for this task).</param>
private void ReJoin(object sender, DoWorkEventArgs ea)
{
BackgroundWorker me = (BackgroundWorker)sender;
while (!me.CancellationPending)
{
try
{
// if this is the first iteration, then the core logic
// is skipped, as the first iteration generally occurs
// right after node Join - allowing a short buffer for
// routing structures to stabilize improves the utility
// of the ReJoin facility.
if (this.m_HasReJoinRun)
{
// first find the successor for the seed node
if (this.m_SeedNode != null)
{
ChordNode seedSuccessor = FindSuccessor(this.m_SeedNode.ID);
// if the successor is not equal to the seed node, something is fishy
if (seedSuccessor.ID != this.m_SeedNode.ID)
{
// if the seed node is still active, re-join the ring to the seed node
ChordInstance instance = ChordServer.GetInstance(this.m_SeedNode);
if (ChordServer.IsInstanceValid(instance))
{
ChordServer.Log(LogLevel.Error, "ReJoin", "Unable to contact initial seed node {0}. Re-Joining...", this.m_SeedNode);
Join(this.m_SeedNode, this.Host, this.Port);
}
// otherwise, in the future, there will be a cache of seed nodes to check/join from...
// as it may be the case that the seed node simply has disconnected from the network.
}
}
}
else
{
// subsequent iterations will go through the core logic
this.m_HasReJoinRun = true;
}
}
catch (Exception e)
{
ChordServer.Log(LogLevel.Error, "Maintenance", "Error occured during ReJoin ({0})", e.Message);
}
// TODO: the delay between iterations, here, is configurable, and
// ideally should be retrieved from configuration or otherwise passed in...
Thread.Sleep(30000);
}
}
/// <summary>
/// Perform equality comparison on ChordNodes on their ID hash value.
/// </summary>
/// <param name="obj">The object to compare this ChordNode to.</param>
/// <returns>True if equal; false, otherwise.</returns>
public override bool Equals(object obj)
{
try
{
ChordNode node = (ChordNode)obj;
return(this.ID == node.ID);
}
catch
{
// Equality operation should not throw...
return(false);
}
}
/// <summary>
/// Find the node that is the rightful owner of a given id.
/// </summary>
/// <param name="id">The id whose successor should be found.</param>
/// <param name="hopCount">The number of network hops taken in finding the successor.</param>
/// <returns>The ChordNode that is the Successor of a given ID value.</returns>
public ChordNode FindSuccessor(UInt64 id, int hopCountIn, out int hopCountOut)
{
// is the local node's successor the rightful owner?
if (ChordServer.IsIDInRange(id, this.ID, this.Successor.ID))
{
hopCountOut = hopCountIn;
return(this.Successor);
}
else
{
// otherwise, find the nearest preceding finger, and ask that node.
ChordNode predNode = FindClosestPrecedingFinger(id);
return(ChordServer.CallFindSuccessor(predNode, id, 0, ++hopCountIn, out hopCountOut));
}
}
/// <summary>
/// Called by the predecessor to a remote node, this acts as a dual heartbeat mechanism and more importantly
/// notification mechanism between predecessor and successor.
/// </summary>
/// <param name="node">A ChordNode instance indicating who the calling node (predecessor) is.</param>
public void Notify(ChordNode node)
{
// if the node has absolutely no predecessor, take
// the first one it finds
if (this.Predecessor == null)
{
this.Predecessor = node;
return;
}
// otherwise, ensure that the predecessor that is calling in
// is indeed valid...
if (ChordServer.IsIDInRange(node.ID, this.Predecessor.ID, this.ID))
{
this.Predecessor = node;
return;
}
}
/// <summary>
/// Get a (local or remote) ChordInstance given a ChordNode.
/// </summary>
/// <param name="node">The ChordNode specifying the node to get an instance of.</param>
/// <returns>A ChordInstance from the specified node, or null if an error is encountered.</returns>
public static ChordInstance GetInstance(ChordNode node)
{
if (node == null)
{
ChordServer.Log(LogLevel.Error, "Navigation", "Invalid Node ({0}).", "Null Argument.");
return(null);
}
try
{
ChordInstance retInstance = (ChordInstance)Activator.GetObject(typeof(ChordInstance), string.Format("tcp://{0}:{1}/chord", node.Host, node.PortNumber));
return(retInstance);
}
catch (Exception e)
{
// perhaps instead we should just pass on the error?
ChordServer.Log(LogLevel.Error, "Navigation", "Unable to activate remote server {0}:{1} ({2}).", node.Host, node.PortNumber, e.Message);
return(null);
}
}
/// <summary>
/// Gets the remote Successor property, given a custom retry count.
/// </summary>
/// <param name="remoteNode">The remote node from which to access the property.</param>
/// <param name="retryCount">The number of times to retry the operation in case of error.</param>
/// <returns>The remote successor, or NULL in case of error.</returns>
public static ChordNode GetSuccessor(ChordNode remoteNode, int retryCount)
{
ChordInstance instance = ChordServer.GetInstance(remoteNode);
try
{
return(instance.Successor);
}
catch (System.Exception ex)
{
ChordServer.Log(LogLevel.Debug, "Remote Accessor", "GetSuccessor error: {0}", ex.Message);
if (retryCount > 0)
{
return(GetSuccessor(remoteNode, --retryCount));
}
else
{
return(null);
}
}
}
/// <summary>
/// Calls AddKey remotely.
/// </summary>
/// <param name="remoteNode">The remote node on which to call AddKey.</param>
/// <param name="value">The string value to add.</param>
/// <param name="retryCount">The number of retries to attempt.</param>
public static void CallAddKey(ChordNode remoteNode, string value, int retryCount)
{
ChordInstance instance = ChordServer.GetInstance(remoteNode);
try
{
instance.AddKey(value);
}
catch (System.Exception ex)
{
ChordServer.Log(LogLevel.Debug, "Remote Invoker", "CallAddKey error: {0}", ex.Message);
if (retryCount > 0)
{
CallAddKey(remoteNode, value, --retryCount);
}
else
{
ChordServer.Log(LogLevel.Debug, "Remote Invoker", "CallAddKey failed - error: {0}", ex.Message);
}
}
}
/// <summary>
/// Calls Notify() remotely, using a default retry value of three.
/// </summary>
/// <param name="remoteNode">The remote on which to call the method.</param>
/// <param name="callingNode">The node to inform the remoteNode of.</param>
/// <param name="retryCount">The number of times to retry the operation in case of error.</param>
/// <returns>True if succeeded, FALSE otherwise.</returns>
public static bool CallNotify(ChordNode remoteNode, ChordNode callingNode, int retryCount)
{
ChordInstance instance = ChordServer.GetInstance(remoteNode);
try
{
instance.Notify(callingNode);
return(true);
}
catch (System.Exception ex)
{
ChordServer.Log(LogLevel.Debug, "Remote Invoker", "CallNotify error: {0}", ex.Message);
if (retryCount > 0)
{
return(CallNotify(remoteNode, callingNode, --retryCount));
}
else
{
return(false);
}
}
}
/// <summary>
/// Calls FindKey remotely.
/// </summary>
/// <param name="remoteNode">The remote node on which to call FindKey.</param>
/// <param name="key">The key to look up.</param>
/// <param name="retryCount">The number of retries to attempt.</param>
/// <returns>The value corresponding to the key, or empty string if not found.</returns>
public static string CallFindKey(ChordNode remoteNode, ulong key, int retryCount)
{
ChordInstance instance = ChordServer.GetInstance(remoteNode);
try
{
return(instance.FindKey(key));
}
catch (System.Exception ex)
{
ChordServer.Log(LogLevel.Debug, "Remote Invoker", "CallFindKey error: {0}", ex.Message);
if (retryCount > 0)
{
return(CallFindKey(remoteNode, key, --retryCount));
}
else
{
ChordServer.Log(LogLevel.Debug, "Remote Invoker", "CallFindKey failed - error: {0}", ex.Message);
return(string.Empty);
}
}
}
/// <summary>
/// Calls FindSuccessor() remotely, using a default retry value of three.
/// </summary>
/// <param name="remoteNode">The remote node on which to call FindSuccessor().</param>
/// <param name="id">The ID to look up.</param>
/// <param name="retryCount">The number of times to retry the operation in case of error.</param>
/// <param name="hopCountIn">The known hopcount prior to calling FindSuccessor on this node.</param>
/// <param name="hopCountOut">The total hopcount of this operation (either returned upwards, or reported for hopcount efficiency validation).</param>
/// <returns>The Successor of ID, or NULL in case of error.</returns>
public static ChordNode CallFindSuccessor(ChordNode remoteNode, UInt64 id, int retryCount, int hopCountIn, out int hopCountOut)
{
ChordInstance instance = ChordServer.GetInstance(remoteNode);
try
{
return(instance.FindSuccessor(id, hopCountIn, out hopCountOut));
}
catch (System.Exception ex)
{
ChordServer.Log(LogLevel.Debug, "Remote Invoker", "CallFindSuccessor error: {0}", ex.Message);
if (retryCount > 0)
{
return(CallFindSuccessor(remoteNode, id, --retryCount, hopCountIn, out hopCountOut));
}
else
{
hopCountOut = hopCountIn;
return(null);
}
}
}
/// <summary>
/// Add a key to the store. Gets a hash of the key, determines
/// the correct owning node, and stores the string value
/// on that node.
/// </summary>
/// <param name="value">The value to add.</param>
public void AddKey(string value)
{
// the key is the hash of the value to
// add to the store, and determines the
// owning NChord node
ulong key = ChordServer.GetHash(value);
// using the local node, determine the correct owning
// node for the data to be stored given the key value
ChordNode owningNode = ChordServer.CallFindSuccessor(key);
if (owningNode != ChordServer.LocalNode)
{
// if this is not the owning node, then call AddKey
// on the actual owning node
ChordServer.CallAddKey(owningNode, value);
}
else
{
// if this is the owning node, then add the
// key to the local data store
this.m_DataStore.Add(key, value);
}
}
/*
* Retry logic:
* The idea behind the retry logic is to provide a simple and common-case reusable call
* in to remote methods or properties. This logic also serendipitously encapsulates and
* simplifies exception handling by performing a bounded number of retries as part of
* exception handling. The retryCount that is passed along as part of the retry logic
* serves as a pleasant way to maintain state across node boundaries (thus enforcing a
* fixed number of N retries for a logical operation, no matter how many nodes the
* operation spans.
*
* Currently, the default retry count is hardcoded; in the future it may be desirable to
* expose this value as a configurable parameter.
*
* Safe access & exception handling pattern:
* Anywhere client or server code needs to make remoting calls, there are typically two
* things people usually do: wrap the call in some sort of exception handling (not doing
* this is generally silly - and is a quick way to wreck whatever application is consuming
* that code upstream), and peform a fixed number of retries in case of transient errors
* (transient errors can be somewhat common when testing with many hundreds of Chord nodes
* running simultaneously on a single OS instance - often, retrying fatal-seeming errors
* can lead to success, reducing the need to exercise (harsher) upstream failure handling).
*
* In almost all cases, upstream code patterns performing these remote access / invocations
* use a single exception handling path; therefore, error is signaled simply via return value
* for simple error-handling (since retry is not needed).
*
*/
/// <summary>
/// Calls Notify() remotely, using a default retry value of three.
/// </summary>
/// <param name="remoteNode">The remote on which to call the method.</param>
/// <param name="callingNode">The node to inform the remoteNode of.</param>
/// <returns>True if succeeded, FALSE otherwise.</returns>
public static bool CallNotify(ChordNode remoteNode, ChordNode callingNode)
{
return(CallNotify(remoteNode, callingNode, 3));
}
/// <summary>
/// Join the current ChordInstance to a chord ring given a seed node.
/// </summary>
/// <param name="seed">Remote node to connect to that is a member of a Chord ring. To start a new Chord ring, a null seed may be provided.</param>
/// <param name="host">The local host name.</param>
/// <param name="port">The tcp port on which this node will listen for calls from other Chord nodes.</param>
/// <returns>true if the Join succeeds; false, otherwise.</returns>
public bool Join(ChordNode seed, string host, int port)
{
// LocalNode is established first to ensure proper logging.
ChordServer.LocalNode = new ChordNode(host, port);
this.m_HasReJoinRun = false;
this.m_SeedNode = seed; // cache the seed node so the ring may re-form itself in case of partition or damage
// safely establish finger table (startvalues and successors, using LocalNode as the default successor)
this.FingerTable = new ChordFingerTable(ChordServer.LocalNode);
// establish successor cache initially with all entries pointing loally
this.SuccessorCache = new ChordNode[3]; // TODO: make this configurable
for (int i = 0; i < this.SuccessorCache.Length; i++)
{
this.SuccessorCache[i] = ChordServer.LocalNode;
}
if (seed != null) // join an existing chord ring
{
ChordServer.Log(LogLevel.Info, "Navigation", "Joining Ring @ {0}:{1}.", seed.Host, seed.PortNumber);
// first, establish successor via seed node
ChordInstance instance = ChordServer.GetInstance(seed);
if (ChordServer.IsInstanceValid(instance))
{
try
{
this.Successor = instance.FindSuccessor(this.ID); // NOTE: this conceivably could be replaced with ChordServer.FindSuccessor()
// disabled: a clever trick that requires only one remote network call is to
// append the successor's successor cache (minus its last entry) to the local
// successor cache, starting at the second entry in the local successor cache.
// during churn, this can break down, so instead the successor cache is populated
// and maintained lazily by maintenance.
// as the successor cache was initialized with the LocalNode as the default
// instance values, "misses" on successor cache entries are gracefully handled by
// simply being forwarded (via the local node) on to the successor node where
// better information may be found and utilized.
//
//this.SuccessorCache = ChordServer.GetSuccessorCache(this.Successor);
}
catch (Exception e)
{
ChordServer.Log(LogLevel.Error, "Navigation", "Error setting Successor Node ({0}).", e.Message);
return(false);
}
}
else
{
ChordServer.Log(LogLevel.Error, "Navigation", "Invalid Node Seed.");
return(false);
}
}
else // start a new ring
{
// not much needs to happen - successor is already established as
// ChordServer.LocalNode,everything else takes place lazily as part of maintenance
ChordServer.Log(LogLevel.Info, "Navigation", "Starting Ring @ {0}:{1}.", this.Host, this.Port);
}
// everything that needs to be populated or kept up-to-date
// lazily is handled via background maintenance threads running periodically
StartMaintenance();
return(true);
}
/// <summary>
/// Calls FindKey() remotely, using a default retry value of three.
/// </summary>
/// <param name="remoteNode">The remote on which to call the method.</param>
/// <param name="key">The key to look up.</param>
/// <returns>The value corresponding to the key, or empty string if not found.</returns>
public static string CallFindKey(ChordNode remoteNode, ulong key)
{
return(CallFindKey(remoteNode, key, 3));
}
/// <summary>
/// Calls AddKey() remotely, using a default retry value of three.
/// </summary>
/// <param name="remoteNode">The remote on which to call the method.</param>
/// <param name="value">The string value to add.</param>
public static void CallAddKey(ChordNode remoteNode, string value)
{
CallAddKey(remoteNode, value, 3);
}
/// <summary>
/// Calls ReplicateKey() remotely, using a default retry value of three.
/// </summary>
/// <param name="remoteNode">The remote on which to call the method.</param>
/// <param name="key">The key to replicate.</param>
/// <param name="value">The string value to replicate.</param>
public static void CallReplicateKey(ChordNode remoteNode, ulong key, string value)
{
CallReplicateKey(remoteNode, key, value, 3);
}
/// <summary>
/// Calls FindSuccessor() remotely, using a default retry value of three. HopCount is ignored.
/// </summary>
/// <param name="remoteNode">The remote on which to call the method.</param>
/// <param name="id">The ID to look up.</param>
/// <returns>The Successor of ID, or NULL in case of error.</returns>
public static ChordNode CallFindSuccessor(ChordNode remoteNode, UInt64 id)
{
int hopCountOut = 0;
return(CallFindSuccessor(remoteNode, id, 3, 0, out hopCountOut));
}
/// <summary>
/// Gets the remote SuccessorCache property, using a default retry value of three.
/// </summary>
/// <param name="remoteNode">The remote from which to access the Successor Cache.</param>
/// <returns>The remote node's successorCache, or NULL in case of error.</returns>
public static ChordNode[] GetSuccessorCache(ChordNode remoteNode)
{
return(GetSuccessorCache(remoteNode, 3));
}
/// <summary>
/// Gets the remote Successor property, using a default retry value of three.
/// </summary>
/// <param name="remoteNode">The remote from which to access the property.</param>
/// <returns>The remote node's successor, or NULL in case of error.</returns>
public static ChordNode GetSuccessor(ChordNode remoteNode)
{
return(GetSuccessor(remoteNode, 3));
}
